From b793f33e59f89f58fef4cf49b2c6df5d9b618b9d Mon Sep 17 00:00:00 2001 From: Mihai Popescu Date: 2026年2月18日 11:48:52 +0200 Subject: [PATCH 1/5] Logs: new materialized view and some UI improvements (#3069) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will prevent internal logs to be added to the task_events_search_table Closes # ## ✅ Checklist - [x] I have followed every step in the [contributing guide](https://github.com/triggerdotdev/trigger.dev/blob/main/CONTRIBUTING.md) - [x] The PR title follows the convention. - [x] I ran and tested the code works --- ## Testing Ran the migration, deleted the old invalid rows and ran new tasks. The undesired logs are not added to the table. --- ## Changelog Updated the MATERIALIZED VIEW to also filter for `trace_id != ''` --------- Co-authored-by: Matt Aitken --- .../app/components/LogLevelTooltipInfo.tsx | 41 +- apps/webapp/app/components/code/CodeBlock.tsx | 6 +- .../app/components/logs/LogDetailView.tsx | 459 +++++------------- apps/webapp/app/components/logs/LogLevel.tsx | 16 + .../app/components/logs/LogsLevelFilter.tsx | 3 + .../app/components/logs/LogsSearchInput.tsx | 67 ++- apps/webapp/app/components/logs/LogsTable.tsx | 38 +- .../app/components/navigation/SideMenu.tsx | 25 +- .../app/components/primitives/DateTime.tsx | 40 +- .../app/components/runs/v3/PacketDisplay.tsx | 4 + .../app/components/runs/v3/SharedFilters.tsx | 6 +- apps/webapp/app/env.server.ts | 13 +- .../presenters/v3/LogsListPresenter.server.ts | 103 ++-- .../route.tsx | 16 +- ...ectParam.env.$envParam.logs.$logId.run.tsx | 202 -------- ...projectParam.env.$envParam.logs.$logId.tsx | 20 +- ...ojects.$projectParam.env.$envParam.logs.ts | 6 +- .../app/services/clickhouseInstance.server.ts | 66 ++- apps/webapp/app/utils/logUtils.ts | 7 +- apps/webapp/tailwind.config.js | 2 +- ...ed_conditions_to_task_events_search_v1.sql | 62 +++ ...018_drop_unused_task_events_v2_indexes.sql | 22 + .../clickhouse/src/client/queryBuilder.ts | 33 ++ internal-packages/clickhouse/src/index.ts | 12 +- .../clickhouse/src/taskEvents.ts | 9 +- 25 files changed, 506 insertions(+), 772 deletions(-) create mode 100644 apps/webapp/app/components/logs/LogLevel.tsx delete mode 100644 apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.run.tsx create mode 100644 internal-packages/clickhouse/schema/017_update_materialized_conditions_to_task_events_search_v1.sql create mode 100644 internal-packages/clickhouse/schema/018_drop_unused_task_events_v2_indexes.sql diff --git a/apps/webapp/app/components/LogLevelTooltipInfo.tsx b/apps/webapp/app/components/LogLevelTooltipInfo.tsx index 6f967af70d1..2a8093af066 100644 --- a/apps/webapp/app/components/LogLevelTooltipInfo.tsx +++ b/apps/webapp/app/components/LogLevelTooltipInfo.tsx @@ -1,7 +1,6 @@ -import { BookOpenIcon } from "@heroicons/react/20/solid"; -import { LinkButton } from "./primitives/Buttons"; import { Header3 } from "./primitives/Headers"; import { Paragraph } from "./primitives/Paragraph"; +import { LogLevel } from "./logs/LogLevel"; export function LogLevelTooltipInfo() { return ( @@ -13,51 +12,45 @@ export function LogLevelTooltipInfo() {
-
- Info +
+ +
+ + Traces and spans representing the execution flow of your tasks. + +
+
+
+
General informational messages about task execution.
-
- Warn +
+
Warning messages indicating potential issues that don't prevent execution.
-
- Error +
+
Error messages for failures and exceptions during task execution.
-
- Debug +
+
Detailed diagnostic information for development and debugging.
-
- Tracing & Spans - - Automatically track the flow of your code through task triggers, attempts, and HTTP - requests. Create custom traces to monitor specific operations. - -
- - Read docs -
); } diff --git a/apps/webapp/app/components/code/CodeBlock.tsx b/apps/webapp/app/components/code/CodeBlock.tsx index 431a02ba35c..8757acc324e 100644 --- a/apps/webapp/app/components/code/CodeBlock.tsx +++ b/apps/webapp/app/components/code/CodeBlock.tsx @@ -68,6 +68,9 @@ type CodeBlockProps = { /** Search term to highlight in the code */ searchTerm?: string; + + /** Whether to wrap the code */ + wrap?: boolean; }; const dimAmount = 0.5; @@ -207,6 +210,7 @@ export const CodeBlock = forwardRef( fileName, rowTitle, searchTerm, + wrap = false, ...props }: CodeBlockProps, ref @@ -215,7 +219,7 @@ export const CodeBlock = forwardRef( const [copied, setCopied] = useState(false); const [modalCopied, setModalCopied] = useState(false); const [isModalOpen, setIsModalOpen] = useState(false); - const [isWrapped, setIsWrapped] = useState(false); + const [isWrapped, setIsWrapped] = useState(wrap); const onCopied = useCallback( (event: React.MouseEvent) => { diff --git a/apps/webapp/app/components/logs/LogDetailView.tsx b/apps/webapp/app/components/logs/LogDetailView.tsx index a5cbb15aeb9..dae58a7b4ce 100644 --- a/apps/webapp/app/components/logs/LogDetailView.tsx +++ b/apps/webapp/app/components/logs/LogDetailView.tsx @@ -1,43 +1,30 @@ -import { XMarkIcon, ArrowTopRightOnSquareIcon, CheckIcon } from "@heroicons/react/20/solid"; -import { Link } from "@remix-run/react"; -import { - type MachinePresetName, - formatDurationMilliseconds, -} from "@trigger.dev/core/v3"; +import { XMarkIcon } from "@heroicons/react/20/solid"; +import type { TaskRunStatus } from "@trigger.dev/database"; import { useEffect, useState } from "react"; import { useTypedFetcher } from "remix-typedjson"; -import { cn } from "~/utils/cn"; -import { Button } from "~/components/primitives/Buttons"; +import { Button, LinkButton } from "~/components/primitives/Buttons"; +import { CopyableText } from "~/components/primitives/CopyableText"; import { DateTimeAccurate } from "~/components/primitives/DateTime"; -import { Header2, Header3 } from "~/components/primitives/Headers"; +import { Header2 } from "~/components/primitives/Headers"; import { Paragraph } from "~/components/primitives/Paragraph"; -import { Spinner } from "~/components/primitives/Spinner"; -import { TabButton, TabContainer } from "~/components/primitives/Tabs"; import * as Property from "~/components/primitives/PropertyTable"; -import { TextLink } from "~/components/primitives/TextLink"; -import { CopyableText } from "~/components/primitives/CopyableText"; -import { SimpleTooltip, InfoIconTooltip } from "~/components/primitives/Tooltip"; +import { Spinner } from "~/components/primitives/Spinner"; +import { SimpleTooltip } from "~/components/primitives/Tooltip"; +import { PacketDisplay } from "~/components/runs/v3/PacketDisplay"; +import { + TaskRunStatusCombo, + descriptionForTaskRunStatus, +} from "~/components/runs/v3/TaskRunStatus"; import { useEnvironment } from "~/hooks/useEnvironment"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; import type { LogEntry } from "~/presenters/v3/LogsListPresenter.server"; -import { getLevelColor } from "~/utils/logUtils"; -import { v3RunSpanPath, v3RunsPath, v3DeploymentVersionPath } from "~/utils/pathBuilder"; import type { loader as logDetailLoader } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId"; -import { TaskRunStatusCombo, descriptionForTaskRunStatus } from "~/components/runs/v3/TaskRunStatus"; -import { MachineLabelCombo } from "~/components/MachineLabelCombo"; -import { EnvironmentCombo } from "~/components/environments/EnvironmentLabel"; -import { RunTag } from "~/components/runs/v3/RunTag"; -import { formatCurrencyAccurate } from "~/utils/numberFormatter"; -import type { TaskRunStatus } from "@trigger.dev/database"; -import { PacketDisplay } from "~/components/runs/v3/PacketDisplay"; -import type { RunContext } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.run"; - -type RunContextData = { - run: RunContext | null; -}; - - +import { cn } from "~/utils/cn"; +import { getLevelColor } from "~/utils/logUtils"; +import { v3RunSpanPath } from "~/utils/pathBuilder"; +import { LogLevel } from "./LogLevel"; +import { ExitIcon } from "~/assets/icons/ExitIcon"; type LogDetailViewProps = { logId: string; // If we have the log entry from the list, we can display it immediately @@ -46,27 +33,38 @@ type LogDetailViewProps = { searchTerm?: string; }; -type TabType = "details" | "run"; - type LogAttributes = Record & { error?: { message?: string; }; }; +function getDisplayMessage(log: { + message: string; + level: string; + attributes?: LogAttributes; +}): string { + let message = log.message ?? ""; + if (log.level === "ERROR") { + const maybeErrorMessage = log.attributes?.error?.message; + if (typeof maybeErrorMessage === "string" && maybeErrorMessage.length> 0) { + message = maybeErrorMessage; + } + } + return message; +} + function formatStringJSON(str: string): string { return str .replace(/\\n/g, "\n") // Converts literal "\n" to newline .replace(/\\t/g, "\t"); // Converts literal "\t" to tab } - export function LogDetailView({ logId, initialLog, onClose, searchTerm }: LogDetailViewProps) { const organization = useOrganization(); const project = useProject(); const environment = useEnvironment(); const fetcher = useTypedFetcher(); - const [activeTab, setActiveTab] = useState("details"); const [error, setError] = useState(null); // Fetch full log details when logId changes @@ -75,7 +73,9 @@ export function LogDetailView({ logId, initialLog, onClose, searchTerm }: LogDet setError(null); fetcher.load( - `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${environment.slug}/logs/${encodeURIComponent(logId)}` + `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${ + environment.slug + }/logs/${encodeURIComponent(logId)}` ); // eslint-disable-next-line react-hooks/exhaustive-deps }, [organization.slug, project.slug, environment.slug, logId]); @@ -93,6 +93,7 @@ export function LogDetailView({ logId, initialLog, onClose, searchTerm }: LogDet const isLoading = fetcher.state === "loading"; const log = fetcher.data ?? initialLog; + const runStatus = fetcher.data?.runStatus; const runPath = v3RunSpanPath( organization, @@ -102,27 +103,6 @@ export function LogDetailView({ logId, initialLog, onClose, searchTerm }: LogDet { spanId: log?.spanId ?? "" } ); - // Handle keyboard shortcuts - useEffect(() => { - const handleKeyDown = (e: KeyboardEvent) => { - const target = e.target as HTMLElement; - if (target && ( - target.tagName === "INPUT" || - target.tagName === "TEXTAREA" || - target.tagName === "SELECT" || - target.contentEditable === "true" - )) { - return; - } - - if (e.key === "Escape") { - onClose(); - } - }; - window.addEventListener("keydown", handleKeyDown); - return () => window.removeEventListener("keydown", handleKeyDown); - }, [onClose, log, runPath, isLoading]); - if (isLoading && !log) { return (
@@ -134,11 +114,16 @@ export function LogDetailView({ logId, initialLog, onClose, searchTerm }: LogDet if (!log) { return (
-
+
Log Details - +
{error ?? "Log not found"} @@ -148,336 +133,126 @@ export function LogDetailView({ logId, initialLog, onClose, searchTerm }: LogDet } return ( -
+
{/* Header */} -
- - {log.level} - - -
- - {/* Tabs */} -
- - setActiveTab("details")} - shortcut={{ key: "d" }} -> - Details - - setActiveTab("run")} - shortcut={{ key: "r" }} -> - Run - - - - - +
+ {getDisplayMessage(log)} +
- - {/* Content */} -
- {activeTab === "details" && ( - - )} - {activeTab === "run" && ( - - )} +
+
); } -function DetailsTab({ log, runPath, searchTerm }: { log: LogEntry; runPath: string; searchTerm?: string }) { - const logWithExtras = log as LogEntry & { +function DetailsTab({ + log, + runPath, + runStatus, + searchTerm, +}: { + log: LogEntry & { attributes?: LogAttributes; }; - - + runPath: string; + runStatus?: TaskRunStatus; + searchTerm?: string; +}) { let beautifiedAttributes: string | null = null; - if (logWithExtras.attributes) { - beautifiedAttributes = JSON.stringify(logWithExtras.attributes, null, 2); + if (log.attributes) { + beautifiedAttributes = JSON.stringify(log.attributes, null, 2); beautifiedAttributes = formatStringJSON(beautifiedAttributes); } const showAttributes = beautifiedAttributes && beautifiedAttributes !== "{}"; - // Determine message to show - let message = log.message ?? ""; - if (log.level === "ERROR") { - const maybeErrorMessage = logWithExtras.attributes?.error?.message; - if (typeof maybeErrorMessage === "string" && maybeErrorMessage.length> 0) { - message = maybeErrorMessage; - } - } + const message = getDisplayMessage(log); return ( - {/* Time */} -
- Timestamp -
- -
-
- - {/* Message */} -
- -
- - {/* Attributes - only available in full log detail */} - {showAttributes && beautifiedAttributes && ( -
- -
- )} - - ); -} - -function RunTab({ log, runPath }: { log: LogEntry; runPath: string }) { - const organization = useOrganization(); - const project = useProject(); - const environment = useEnvironment(); - const fetcher = useTypedFetcher(); - - // Fetch run details when tab is active - useEffect(() => { - if (!log.runId) return; - - fetcher.load( - `/resources/orgs/${organization.slug}/projects/${project.slug}/env/${environment.slug}/logs/${encodeURIComponent(log.id)}/run?runId=${encodeURIComponent(log.runId)}` - ); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [organization.slug, project.slug, environment.slug, log.id, log.runId]); - - const isLoading = fetcher.state === "loading"; - const runData = fetcher.data?.run; - - if (isLoading) { - return ( -
- -
- ); - } - - if (!runData) { - return ( -
- Run not found in database. -
- ); - } - - return ( -
Run ID - - - - - - Status - - } - content={descriptionForTaskRunStatus(runData.status as TaskRunStatus)} - disableHoverableContent - /> - - - - - Task - - + + + View full run + - {runData.rootRun && ( + {runStatus && ( - Root and parent run + Status - } + content={descriptionForTaskRunStatus(runStatus)} + disableHoverableContent + className="mt-1" /> )} - {runData.batch && ( - - Batch - - - - - )} - - - Version - - {runData.version ? ( - environment.type === "DEVELOPMENT" ? ( - - ) : ( - - - - } - content={"Jump to deployment"} - /> - ) - ) : ( - - Never started - - - )} - - - - - Test run - - {runData.isTest ? : "–"} - - - - - Environment - - - - - - Queue + Task -
Name: {runData.queue}
-
Concurrency key: {runData.concurrencyKey ? runData.concurrencyKey : "–"}
- - - - {runData.tags && runData.tags.length> 0 && ( - - Tags - -
- {runData.tags.map((tag: string) => ( - - ))} -
- - - )} - - - Machine - - {runData.machinePreset ? ( - - ) : ( - "–" - )} + - Run invocation cost + Level - {runData.baseCostInCents> 0 - ? formatCurrencyAccurate(runData.baseCostInCents / 100) - : "–"} + - Compute cost + Timestamp - {runData.costInCents> 0 ? formatCurrencyAccurate(runData.costInCents / 100) : "–"} + + - - Total cost - - {runData.costInCents> 0 || runData.baseCostInCents> 0 - ? formatCurrencyAccurate((runData.baseCostInCents + runData.costInCents) / 100) - : "–"} - - + {/* Message */} +
+ +
- - Usage duration - - {runData.usageDurationMs> 0 - ? formatDurationMilliseconds(runData.usageDurationMs, { style: "short" }) - : "–"} - - - -
+ {/* Attributes - only available in full log detail */} + {showAttributes && beautifiedAttributes && ( +
+ +
+ )} + ); } - diff --git a/apps/webapp/app/components/logs/LogLevel.tsx b/apps/webapp/app/components/logs/LogLevel.tsx new file mode 100644 index 00000000000..3c2c38301ff --- /dev/null +++ b/apps/webapp/app/components/logs/LogLevel.tsx @@ -0,0 +1,16 @@ +import { cn } from "~/utils/cn"; +import { getLevelColor } from "~/utils/logUtils"; +import type { LogEntry } from "~/presenters/v3/LogsListPresenter.server"; + +export function LogLevel({ level }: { level: LogEntry["level"] }) { + return ( + + {level} + + ); +} diff --git a/apps/webapp/app/components/logs/LogsLevelFilter.tsx b/apps/webapp/app/components/logs/LogsLevelFilter.tsx index 8c2abf64f25..947bef88fcc 100644 --- a/apps/webapp/app/components/logs/LogsLevelFilter.tsx +++ b/apps/webapp/app/components/logs/LogsLevelFilter.tsx @@ -16,6 +16,7 @@ import type { LogLevel } from "~/presenters/v3/LogsListPresenter.server"; import { cn } from "~/utils/cn"; const allLogLevels: { level: LogLevel; label: string; color: string }[] = [ + { level: "TRACE", label: "Trace", color: "text-purple-400" }, { level: "INFO", label: "Info", color: "text-blue-400" }, { level: "WARN", label: "Warning", color: "text-warning" }, { level: "ERROR", label: "Error", color: "text-error" }, @@ -33,6 +34,8 @@ function getLevelBadgeColor(level: LogLevel): string { return "text-error bg-error/10 border-error/20"; case "WARN": return "text-warning bg-warning/10 border-warning/20"; + case "TRACE": + return "text-purple-400 bg-purple-500/10 border-purple-500/20"; case "DEBUG": return "text-charcoal-400 bg-charcoal-700 border-charcoal-600"; case "INFO": diff --git a/apps/webapp/app/components/logs/LogsSearchInput.tsx b/apps/webapp/app/components/logs/LogsSearchInput.tsx index fd539f66ae2..58316cead88 100644 --- a/apps/webapp/app/components/logs/LogsSearchInput.tsx +++ b/apps/webapp/app/components/logs/LogsSearchInput.tsx @@ -1,55 +1,49 @@ import { MagnifyingGlassIcon, XMarkIcon } from "@heroicons/react/20/solid"; -import { useNavigate } from "@remix-run/react"; import { motion } from "framer-motion"; import { useCallback, useEffect, useRef, useState } from "react"; import { Input } from "~/components/primitives/Input"; import { ShortcutKey } from "~/components/primitives/ShortcutKey"; import { cn } from "~/utils/cn"; import { useOptimisticLocation } from "~/hooks/useOptimisticLocation"; +import { useSearchParams } from "~/hooks/useSearchParam"; export function LogsSearchInput() { const location = useOptimisticLocation(); - const navigate = useNavigate(); const inputRef = useRef(null); + const { value, replace, del } = useSearchParams(); + // Get initial search value from URL - const searchParams = new URLSearchParams(location.search); - const initialSearch = searchParams.get("search") ?? ""; + const initialSearch = value("search") ?? ""; const [text, setText] = useState(initialSearch); const [isFocused, setIsFocused] = useState(false); // Update text when URL search param changes (only when not focused to avoid overwriting user input) useEffect(() => { - const params = new URLSearchParams(location.search); - const urlSearch = params.get("search") ?? ""; + const urlSearch = value("search") ?? ""; if (urlSearch !== text && !isFocused) { setText(urlSearch); } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [location.search]); + }, [value, text, isFocused]); const handleSubmit = useCallback(() => { - const params = new URLSearchParams(location.search); if (text.trim()) { - params.set("search", text.trim()); + replace({ search: text.trim() }); } else { - params.delete("search"); + del("search"); } - // Reset cursor when searching - params.delete("cursor"); - params.delete("direction"); - navigate(`${location.pathname}?${params.toString()}`, { replace: true }); - }, [text, location.pathname, location.search, navigate]); + }, [text, replace, del]); - const handleClear = useCallback(() => { - setText(""); - const params = new URLSearchParams(location.search); - params.delete("search"); - params.delete("cursor"); - params.delete("direction"); - navigate(`${location.pathname}?${params.toString()}`, { replace: true }); - }, [location.pathname, location.search, navigate]); + const handleClear = useCallback( + (e: React.MouseEvent) => { + e.preventDefault(); + e.stopPropagation(); + setText(""); + del(["search", "cursor", "direction"]); + }, + [del] + ); return (
@@ -71,7 +65,7 @@ export function LogsSearchInput() { value={text} onChange={(e) => setText(e.target.value)} fullWidth - className={cn(isFocused && "placeholder:text-text-dimmed/70")} + className={cn("", isFocused && "placeholder:text-text-dimmed/70")} onKeyDown={(e) => { if (e.key === "Enter") { e.preventDefault(); @@ -86,22 +80,21 @@ export function LogsSearchInput() { icon={} accessory={ text.length> 0 ? ( - +
+ + +
) : undefined } /> - - {text.length> 0 && ( - - )}
); } diff --git a/apps/webapp/app/components/logs/LogsTable.tsx b/apps/webapp/app/components/logs/LogsTable.tsx index 2f6894a03e6..0aedc4d706d 100644 --- a/apps/webapp/app/components/logs/LogsTable.tsx +++ b/apps/webapp/app/components/logs/LogsTable.tsx @@ -2,16 +2,17 @@ import { ArrowPathIcon, ArrowTopRightOnSquareIcon } from "@heroicons/react/20/so import { Link } from "@remix-run/react"; import { useEffect, useRef, useState } from "react"; import { cn } from "~/utils/cn"; -import { Button } from "~/components/primitives/Buttons"; +import { Button, LinkButton } from "~/components/primitives/Buttons"; import { useEnvironment } from "~/hooks/useEnvironment"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; import type { LogEntry } from "~/presenters/v3/LogsListPresenter.server"; -import { getLevelColor, highlightSearchText } from "~/utils/logUtils"; +import { highlightSearchText } from "~/utils/logUtils"; import { v3RunSpanPath } from "~/utils/pathBuilder"; import { DateTimeAccurate } from "../primitives/DateTime"; import { Paragraph } from "../primitives/Paragraph"; import { Spinner } from "../primitives/Spinner"; +import { LogLevel } from "./LogLevel"; import { TruncatedCopyableValue } from "../primitives/TruncatedCopyableValue"; import { LogLevelTooltipInfo } from "~/components/LogLevelTooltipInfo"; import { @@ -48,14 +49,14 @@ function getLevelBoxShadow(level: LogEntry["level"]): string { return "inset 2px 0 0 0 rgb(234, 179, 8)"; case "INFO": return "inset 2px 0 0 0 rgb(59, 130, 246)"; + case "TRACE": + return "inset 2px 0 0 0 rgb(168, 85, 247)"; case "DEBUG": default: return "none"; } } - - export function LogsTable({ logs, searchTerm, @@ -162,7 +163,7 @@ export function LogsTable({ boxShadow: getLevelBoxShadow(log.level), }} > - + @@ -171,14 +172,7 @@ export function LogsTable({ {log.taskIdentifier}
- - {log.level} - +
@@ -188,11 +182,13 @@ export function LogsTable({
- - + + View run + } /> @@ -233,11 +229,7 @@ function BlankState({ isLoading, onRefresh }: { isLoading?: boolean; onRefresh?: No logs match your filters. Try refreshing or modifying your filters.
-
diff --git a/apps/webapp/app/components/navigation/SideMenu.tsx b/apps/webapp/app/components/navigation/SideMenu.tsx index 241bb8aea2b..adedf7b589d 100644 --- a/apps/webapp/app/components/navigation/SideMenu.tsx +++ b/apps/webapp/app/components/navigation/SideMenu.tsx @@ -433,18 +433,7 @@ export function SideMenu({ data-action="deployments" isCollapsed={isCollapsed} /> - {(user.admin || user.isImpersonating || featureFlags.hasLogsPageAccess) && ( - } - isCollapsed={isCollapsed} - /> - )} + + {(user.admin || user.isImpersonating || featureFlags.hasLogsPageAccess) && ( + } + isCollapsed={isCollapsed} + /> + )} {formattedDateTime.replace(/\s/g, String.fromCharCode(32))}; + return ( + + {formattedDateTime.replace(/\s/g, String.fromCharCode(32))} + + ); }; // Helper function to check if two dates are on the same day @@ -270,14 +274,18 @@ const DateTimeAccurateInner = ({ return hideDate ? formatTimeOnly(realDate, displayTimeZone, locales, hour12) : realPrevDate - ? isSameDay(realDate, realPrevDate) - ? formatTimeOnly(realDate, displayTimeZone, locales, hour12) - : formatDateTimeAccurate(realDate, displayTimeZone, locales, hour12) - : formatDateTimeAccurate(realDate, displayTimeZone, locales, hour12); + ? isSameDay(realDate, realPrevDate) + ? formatTimeOnly(realDate, displayTimeZone, locales, hour12) + : formatDateTimeAccurate(realDate, displayTimeZone, locales, hour12) + : formatDateTimeAccurate(realDate, displayTimeZone, locales, hour12); }, [realDate, displayTimeZone, locales, hour12, hideDate, previousDate]); if (!showTooltip) - return {formattedDateTime.replace(/\s/g, String.fromCharCode(32))}; + return ( + + {formattedDateTime.replace(/\s/g, String.fromCharCode(32))} + + ); const tooltipContent = ( {formattedDateTime.replace(/\s/g, String.fromCharCode(32))}} + button={ + + {formattedDateTime.replace(/\s/g, String.fromCharCode(32))} + + } content={tooltipContent} side="right" asChild={true} @@ -326,9 +338,13 @@ function formatDateTimeAccurate( locales: string[], hour12: boolean = true ): string { - const formattedDateTime = new Intl.DateTimeFormat(locales, { + const datePart = new Intl.DateTimeFormat(locales, { month: "short", day: "numeric", + timeZone, + }).format(date); + + const timePart = new Intl.DateTimeFormat(locales, { hour: "numeric", minute: "numeric", second: "numeric", @@ -338,7 +354,7 @@ function formatDateTimeAccurate( hour12, }).format(date); - return formattedDateTime; + return `${datePart} ${timePart}`; } export const DateTimeShort = ({ date, hour12 = true }: DateTimeProps) => { @@ -347,7 +363,11 @@ export const DateTimeShort = ({ date, hour12 = true }: DateTimeProps) => { const realDate = typeof date === "string" ? new Date(date) : date; const formattedDateTime = formatDateTimeShort(realDate, userTimeZone, locales, hour12); - return {formattedDateTime.replace(/\s/g, String.fromCharCode(32))}; + return ( + + {formattedDateTime.replace(/\s/g, String.fromCharCode(32))} + + ); }; function formatDateTimeShort( diff --git a/apps/webapp/app/components/runs/v3/PacketDisplay.tsx b/apps/webapp/app/components/runs/v3/PacketDisplay.tsx index 2d6b6e65be6..82deae8a1be 100644 --- a/apps/webapp/app/components/runs/v3/PacketDisplay.tsx +++ b/apps/webapp/app/components/runs/v3/PacketDisplay.tsx @@ -12,11 +12,13 @@ export function PacketDisplay({ dataType, title, searchTerm, + wrap, }: { data: string; dataType: string; title: string; searchTerm?: string; + wrap?: boolean; }) { switch (dataType) { case "application/store": { @@ -54,6 +56,7 @@ export function PacketDisplay({ showLineNumbers={false} showTextWrapping searchTerm={searchTerm} + wrap={wrap} /> ); } @@ -67,6 +70,7 @@ export function PacketDisplay({ showLineNumbers={false} showTextWrapping searchTerm={searchTerm} + wrap={wrap} /> ); } diff --git a/apps/webapp/app/components/runs/v3/SharedFilters.tsx b/apps/webapp/app/components/runs/v3/SharedFilters.tsx index 7a5c4fdba93..5538c628178 100644 --- a/apps/webapp/app/components/runs/v3/SharedFilters.tsx +++ b/apps/webapp/app/components/runs/v3/SharedFilters.tsx @@ -219,7 +219,7 @@ export function timeFilterFromTo(props: { from?: string | number; to?: string | number; defaultPeriod: string; -}): { from: Date; to: Date } { +}): { from: Date; to: Date; isDefault: boolean } { const time = timeFilters(props); const periodMs = time.period ? parse(time.period) : undefined; @@ -228,6 +228,7 @@ export function timeFilterFromTo(props: { return { from: new Date(Date.now() - periodMs), to: new Date(), + isDefault: time.isDefault, }; } @@ -235,6 +236,7 @@ export function timeFilterFromTo(props: { return { from: time.from, to: time.to, + isDefault: time.isDefault, }; } @@ -242,6 +244,7 @@ export function timeFilterFromTo(props: { return { from: time.from, to: new Date(), + isDefault: time.isDefault, }; } @@ -249,6 +252,7 @@ export function timeFilterFromTo(props: { return { from: new Date(Date.now() - defaultPeriodMs), to: time.to ?? new Date(), + isDefault: time.isDefault, }; } diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 5a321c58b64..6d3e6fbe3d1 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -1180,7 +1180,7 @@ const EnvironmentSchema = z CLICKHOUSE_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"), CLICKHOUSE_COMPRESSION_REQUEST: z.string().default("1"), - // Logs List Query Settings (for paginated log views) + // Logs Query Settings CLICKHOUSE_LOGS_LIST_MAX_MEMORY_USAGE: z.coerce.number().int().default(1_000_000_000), CLICKHOUSE_LOGS_LIST_MAX_BYTES_BEFORE_EXTERNAL_SORT: z.coerce .number() @@ -1190,14 +1190,15 @@ const EnvironmentSchema = z CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ: z.coerce.number().int().default(10_000_000), CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME: z.coerce.number().int().default(120), - // Logs Detail Query Settings (for single log views) - CLICKHOUSE_LOGS_DETAIL_MAX_MEMORY_USAGE: z.coerce.number().int().default(64_000_000), - CLICKHOUSE_LOGS_DETAIL_MAX_THREADS: z.coerce.number().int().default(2), - CLICKHOUSE_LOGS_DETAIL_MAX_EXECUTION_TIME: z.coerce.number().int().default(60), - // Query feature flag QUERY_FEATURE_ENABLED: z.string().default("1"), + // Logs page ClickHouse URL (for logs queries) + LOGS_CLICKHOUSE_URL: z + .string() + .optional() + .transform((v) => v ?? process.env.CLICKHOUSE_URL), + // Query page ClickHouse limits (for TSQL queries) QUERY_CLICKHOUSE_URL: z .string() diff --git a/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts b/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts index 545bca5cce7..8a3bf692b5b 100644 --- a/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts @@ -1,13 +1,11 @@ import { z } from "zod"; -import { type ClickHouse } from "@internal/clickhouse"; -import { - type PrismaClientOrTransaction, -} from "@trigger.dev/database"; +import { type ClickHouse, type WhereCondition } from "@internal/clickhouse"; +import { type PrismaClientOrTransaction } from "@trigger.dev/database"; import { EVENT_STORE_TYPES, getConfiguredEventRepository } from "~/v3/eventRepository/index.server"; import parseDuration from "parse-duration"; import { type Direction } from "~/components/ListPagination"; -import { timeFilters } from "~/components/runs/v3/SharedFilters"; +import { timeFilterFromTo, timeFilters } from "~/components/runs/v3/SharedFilters"; import { findDisplayableEnvironment } from "~/models/runtimeEnvironment.server"; import { getAllTaskIdentifiers } from "~/models/task.server"; import { ServiceValidationError } from "~/v3/services/baseService.server"; @@ -28,14 +26,9 @@ type ErrorAttributes = { }; function escapeClickHouseString(val: string): string { - return val - .replace(/\\/g, "\\\\") - .replace(/\//g, "\\/") - .replace(/%/g, "\\%") - .replace(/_/g, "\\_"); + return val.replace(/\\/g, "\\\\").replace(/\//g, "\\/").replace(/%/g, "\\%").replace(/_/g, "\\_"); } - export type LogsListOptions = { userId?: string; projectId: string; @@ -115,10 +108,12 @@ function decodeCursor(cursor: string): LogCursor | null { // Convert display level to ClickHouse kinds and statuses function levelToKindsAndStatuses(level: LogLevel): { kinds?: string[]; statuses?: string[] } { switch (level) { + case "TRACE": + return { kinds: ["SPAN"] }; case "DEBUG": return { kinds: ["LOG_DEBUG"] }; case "INFO": - return { kinds: ["LOG_INFO", "LOG_LOG", "SPAN"] }; + return { kinds: ["LOG_INFO", "LOG_LOG"] }; case "WARN": return { kinds: ["LOG_WARN"] }; case "ERROR": @@ -153,24 +148,16 @@ export class LogsListPresenter extends BasePresenter { retentionLimitDays, }: LogsListOptions ) { - const time = timeFilters({ + const time = timeFilterFromTo({ period, from, to, - defaultPeriod, + defaultPeriod: defaultPeriod ?? "1h", }); let effectiveFrom = time.from; let effectiveTo = time.to; - if (!effectiveFrom && !effectiveTo && time.period) { - const periodMs = parseDuration(time.period); - if (periodMs) { - effectiveFrom = new Date(Date.now() - periodMs); - effectiveTo = new Date(); - } - } - // Apply retention limit if provided let wasClampedByRetention = false; if (retentionLimitDays !== undefined && effectiveFrom) { @@ -236,6 +223,11 @@ export class LogsListPresenter extends BasePresenter { const queryBuilder = this.clickhouse.taskEventsSearch.logsListQueryBuilder(); + // This should be removed once we clear the old inserts, 30 DAYS, the materialized view excludes events without trace_id) + queryBuilder.where("trace_id != ''", { + environmentId, + }); + queryBuilder.where("environment_id = {environmentId: String}", { environmentId, }); @@ -245,11 +237,10 @@ export class LogsListPresenter extends BasePresenter { }); queryBuilder.where("project_id = {projectId: String}", { projectId }); - if (effectiveFrom) { - queryBuilder.where("triggered_timestamp>= {triggeredAtStart: DateTime64(3)}", { - triggeredAtStart: convertDateToClickhouseDateTime(effectiveFrom), - }); + queryBuilder.where("triggered_timestamp>= {triggeredAtStart: DateTime64(3)}", { + triggeredAtStart: convertDateToClickhouseDateTime(effectiveFrom), + }); } if (effectiveTo) { @@ -278,50 +269,43 @@ export class LogsListPresenter extends BasePresenter { queryBuilder.where( "(lower(message) like {searchPattern: String} OR lower(attributes_text) like {searchPattern: String})", { - searchPattern: `%${searchTerm}%` + searchPattern: `%${searchTerm}%`, } ); } if (levels && levels.length> 0) { - const conditions: string[] = []; - const params: Record = {}; + const conditions: WhereCondition[] = []; - for (const level of levels) { - const filter = levelToKindsAndStatuses(level); - const levelConditions: string[] = []; + for (let i = 0; i < levels.length; i++) { + const filter = levelToKindsAndStatuses(levels[i]); if (filter.kinds && filter.kinds.length> 0) { - const kindsKey = `kinds_${level}`; - let kindCondition = `kind IN {${kindsKey}: Array(String)}`; - - - kindCondition += ` AND status NOT IN {excluded_statuses: Array(String)}`; - params["excluded_statuses"] = ["ERROR", "CANCELLED"]; - - - levelConditions.push(kindCondition); - params[kindsKey] = filter.kinds; + conditions.push({ + clause: `kind IN {kinds_${i}: Array(String)} AND status NOT IN {excluded_statuses: Array(String)}`, + params: { + [`kinds_${i}`]: filter.kinds, + excluded_statuses: ["ERROR", "CANCELLED"], + }, + }); } if (filter.statuses && filter.statuses.length> 0) { - const statusesKey = `statuses_${level}`; - levelConditions.push(`status IN {${statusesKey}: Array(String)}`); - params[statusesKey] = filter.statuses; - } - - if (levelConditions.length> 0) { - conditions.push(`(${levelConditions.join(" OR ")})`); + conditions.push({ + clause: `status IN {statuses_${i}: Array(String)}`, + params: { [`statuses_${i}`]: filter.statuses }, + }); } } - if (conditions.length> 0) { - queryBuilder.where(`(${conditions.join(" OR ")})`, params); - } + queryBuilder.whereOr(conditions); } - // Cursor pagination using explicit lexicographic comparison - // Must mirror the ORDER BY columns: (organization_id, environment_id, triggered_timestamp, trace_id) + // Cursor-based pagination using lexicographic comparison on (triggered_timestamp, trace_id). + // Since ORDER BY is DESC, "next page" means rows that sort *after* the cursor, i.e. less-than. + // The OR handles the tiebreaker: rows with an earlier timestamp always qualify, and rows + // with the *same* timestamp only qualify if their trace_id is also smaller. + // Equivalent to: WHERE (triggered_timestamp, trace_id) < (cursor.triggered_timestamp, cursor.trace_id) const decodedCursor = cursor ? decodeCursor(cursor) : null; if (decodedCursor) { queryBuilder.where( @@ -423,10 +407,13 @@ export class LogsListPresenter extends BasePresenter { hasFilters, hasAnyLogs: transformedLogs.length> 0, searchTerm: search, - retention: retentionLimitDays !== undefined ? { - limitDays: retentionLimitDays, - wasClamped: wasClampedByRetention, - } : undefined, + retention: + retentionLimitDays !== undefined + ? { + limitDays: retentionLimitDays, + wasClamped: wasClampedByRetention, + } + : undefined, }; } } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx index 9c2eccc8ea0..325df8b386b 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx @@ -16,7 +16,7 @@ import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { LogsListPresenter, LogEntry } from "~/presenters/v3/LogsListPresenter.server"; import type { LogLevel } from "~/utils/logUtils"; import { $replica, prisma } from "~/db.server"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { logsClickhouseClient } from "~/services/clickhouseInstance.server"; import { NavBar, PageTitle } from "~/components/primitives/PageHeader"; import { PageBody, PageContainer } from "~/components/layout/AppLayout"; import { Suspense, useCallback, useEffect, useMemo, useRef, useState, useTransition } from "react"; @@ -40,7 +40,7 @@ import { Button } from "~/components/primitives/Buttons"; import { FEATURE_FLAG, validateFeatureFlagValue } from "~/v3/featureFlags.server"; // Valid log levels for filtering -const validLevels: LogLevel[] = ["DEBUG", "INFO", "WARN", "ERROR"]; +const validLevels: LogLevel[] = ["TRACE", "DEBUG", "INFO", "WARN", "ERROR"]; function parseLevelsFromUrl(url: URL): LogLevel[] | undefined { const levelParams = url.searchParams.getAll("levels").filter((v) => v.length> 0); @@ -134,7 +134,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const plan = await getCurrentPlan(project.organizationId); const retentionLimitDays = plan?.v3Subscription?.plan?.limits.logRetentionDays.number ?? 30; - const presenter = new LogsListPresenter($replica, clickhouseClient); + const presenter = new LogsListPresenter($replica, logsClickhouseClient); const listPromise = presenter .call(project.organizationId, environment.id, { @@ -322,7 +322,10 @@ function LogsList({ const [nextCursor, setNextCursor] = useState(list.pagination.next); // Selected log state - managed locally to avoid triggering navigation - const [selectedLogId, setSelectedLogId] = useState(); + const [selectedLogId, setSelectedLogId] = useState(() => { + const params = new URLSearchParams(location.search); + return params.get("log") ?? undefined; + }); // Track which filter state (search params) the current fetcher request corresponds to const fetcherFilterStateRef = useRef(location.search); @@ -333,8 +336,9 @@ function LogsList({ useEffect(() => { setAccumulatedLogs([]); setNextCursor(undefined); - // Close side panel when filters change to avoid showing a log that's no longer visible - setSelectedLogId(undefined); + // Preserve log selection from URL param, clear if not present + const params = new URLSearchParams(location.search); + setSelectedLogId(params.get("log") ?? undefined); }, [location.search]); // Populate accumulated logs when new data arrives diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.run.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.run.tsx deleted file mode 100644 index 15986c305af..00000000000 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.run.tsx +++ /dev/null @@ -1,202 +0,0 @@ -import { type LoaderFunctionArgs } from "@remix-run/server-runtime"; -import { json } from "@remix-run/node"; -import { z } from "zod"; -import { MachinePresetName } from "@trigger.dev/core/v3"; -import { requireUserId } from "~/services/session.server"; -import { EnvironmentParamSchema } from "~/utils/pathBuilder"; -import { findProjectBySlug } from "~/models/project.server"; -import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; -import { $replica } from "~/db.server"; -import type { TaskRunStatus } from "@trigger.dev/database"; - -// Valid TaskRunStatus values -const VALID_TASK_RUN_STATUSES = [ - "PENDING", - "QUEUED", - "EXECUTING", - "WAITING_FOR_EXECUTION", - "WAITING", - "COMPLETED_SUCCESSFULLY", - "COMPLETED_WITH_ERRORS", - "SYSTEM_FAILURE", - "FAILURE", - "CANCELED", -] as const; - -// Schema for validating run context data -export const RunContextSchema = z.object({ - id: z.string(), - friendlyId: z.string(), - taskIdentifier: z.string(), - status: z.enum(VALID_TASK_RUN_STATUSES), - createdAt: z.string().datetime(), - startedAt: z.string().datetime().optional(), - completedAt: z.string().datetime().optional(), - isTest: z.boolean(), - tags: z.array(z.string()), - queue: z.string(), - concurrencyKey: z.string().nullable(), - usageDurationMs: z.number(), - costInCents: z.number(), - baseCostInCents: z.number(), - machinePreset: MachinePresetName.nullable(), - version: z.string().optional(), - rootRun: z - .object({ - friendlyId: z.string(), - taskIdentifier: z.string(), - }) - .nullable(), - parentRun: z - .object({ - friendlyId: z.string(), - taskIdentifier: z.string(), - }) - .nullable(), - batch: z - .object({ - friendlyId: z.string(), - }) - .nullable(), - schedule: z - .object({ - friendlyId: z.string(), - }) - .nullable(), -}); - -export type RunContext = z.infer; - -// Fetch run context for a log entry -export const loader = async ({ request, params }: LoaderFunctionArgs) => { - const userId = await requireUserId(request); - const { projectParam, organizationSlug, envParam, logId } = { - ...EnvironmentParamSchema.parse(params), - logId: params.logId, - }; - - if (!logId) { - throw new Response("Log ID is required", { status: 400 }); - } - - const project = await findProjectBySlug(organizationSlug, projectParam, userId); - if (!project) { - throw new Response("Project not found", { status: 404 }); - } - - const environment = await findEnvironmentBySlug(project.id, envParam, userId); - if (!environment) { - throw new Response("Environment not found", { status: 404 }); - } - - // Parse the logId to extract runId - // Log ID format: traceId::spanId::runId::startTime (base64 encoded or plain) - const url = new URL(request.url); - const runId = url.searchParams.get("runId"); - - if (!runId) { - throw new Response("Run ID is required", { status: 400 }); - } - - // Fetch run details from Postgres - const run = await $replica.taskRun.findFirst({ - select: { - id: true, - friendlyId: true, - taskIdentifier: true, - status: true, - createdAt: true, - startedAt: true, - completedAt: true, - isTest: true, - runTags: true, - queue: true, - concurrencyKey: true, - usageDurationMs: true, - costInCents: true, - baseCostInCents: true, - machinePreset: true, - scheduleId: true, - lockedToVersion: { - select: { - version: true, - }, - }, - rootTaskRun: { - select: { - friendlyId: true, - taskIdentifier: true, - }, - }, - parentTaskRun: { - select: { - friendlyId: true, - taskIdentifier: true, - }, - }, - batch: { - select: { - friendlyId: true, - }, - }, - }, - where: { - friendlyId: runId, - runtimeEnvironmentId: environment.id, - }, - }); - - if (!run) { - return json({ run: null }); - } - - // Fetch schedule if scheduleId exists - let schedule: { friendlyId: string } | null = null; - if (run.scheduleId) { - const scheduleData = await $replica.taskSchedule.findFirst({ - select: { friendlyId: true }, - where: { id: run.scheduleId }, - }); - schedule = scheduleData; - } - - const runData = { - id: run.id, - friendlyId: run.friendlyId, - taskIdentifier: run.taskIdentifier, - status: run.status, - createdAt: run.createdAt.toISOString(), - startedAt: run.startedAt?.toISOString(), - completedAt: run.completedAt?.toISOString(), - isTest: run.isTest, - tags: run.runTags, - queue: run.queue, - concurrencyKey: run.concurrencyKey, - usageDurationMs: run.usageDurationMs, - costInCents: run.costInCents, - baseCostInCents: run.baseCostInCents, - machinePreset: run.machinePreset, - version: run.lockedToVersion?.version, - rootRun: run.rootTaskRun - ? { - friendlyId: run.rootTaskRun.friendlyId, - taskIdentifier: run.rootTaskRun.taskIdentifier, - } - : null, - parentRun: run.parentTaskRun - ? { - friendlyId: run.parentTaskRun.friendlyId, - taskIdentifier: run.parentTaskRun.taskIdentifier, - } - : null, - batch: run.batch ? { friendlyId: run.batch.friendlyId } : null, - schedule: schedule, - }; - - // Validate the run data - const validatedRun = RunContextSchema.parse(runData); - - return json({ - run: validatedRun, - }); -}; diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.tsx index f04e706d4e5..f862ced6b05 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.tsx @@ -1,13 +1,14 @@ import { type LoaderFunctionArgs } from "@remix-run/server-runtime"; import { typedjson } from "remix-typedjson"; import { z } from "zod"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { logsClickhouseClient } from "~/services/clickhouseInstance.server"; import { requireUserId } from "~/services/session.server"; import { LogDetailPresenter } from "~/presenters/v3/LogDetailPresenter.server"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { $replica } from "~/db.server"; import { ServiceValidationError } from "~/v3/services/baseService.server"; +import type { TaskRunStatus } from "@trigger.dev/database"; const LogIdParamsSchema = z.object({ organizationSlug: z.string(), @@ -42,7 +43,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const [traceId, spanId, , startTime] = parts; - const presenter = new LogDetailPresenter($replica, clickhouseClient); + const presenter = new LogDetailPresenter($replica, logsClickhouseClient); let result; try { @@ -65,5 +66,18 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { throw new Response("Log not found", { status: 404 }); } - return typedjson(result); + // Look up the run status from Postgres + let runStatus: TaskRunStatus | undefined; + if (result.runId) { + const run = await $replica.taskRun.findFirst({ + select: { status: true }, + where: { + friendlyId: result.runId, + runtimeEnvironmentId: environment.id, + }, + }); + runStatus = run?.status; + } + + return typedjson({ ...result, runStatus }); }; diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts index cac4c0f7025..66ddebe4e2a 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts @@ -6,11 +6,11 @@ import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { LogsListPresenter, type LogLevel, LogsListOptionsSchema } from "~/presenters/v3/LogsListPresenter.server"; import { $replica } from "~/db.server"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { logsClickhouseClient } from "~/services/clickhouseInstance.server"; import { getCurrentPlan } from "~/services/platform.v3.server"; // Valid log levels for filtering -const validLevels: LogLevel[] = ["DEBUG", "INFO", "WARN", "ERROR"]; +const validLevels: LogLevel[] = ["TRACE", "DEBUG", "INFO", "WARN", "ERROR"]; function parseLevelsFromUrl(url: URL): LogLevel[] | undefined { const levelParams = url.searchParams.getAll("levels").filter((v) => v.length> 0); @@ -69,7 +69,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { retentionLimitDays, }) as any; // Validated by LogsListOptionsSchema at runtime - const presenter = new LogsListPresenter($replica, clickhouseClient); + const presenter = new LogsListPresenter($replica, logsClickhouseClient); const result = await presenter.call(project.organizationId, environment.id, options); return json({ diff --git a/apps/webapp/app/services/clickhouseInstance.server.ts b/apps/webapp/app/services/clickhouseInstance.server.ts index 7818129b421..f88b3baaaed 100644 --- a/apps/webapp/app/services/clickhouseInstance.server.ts +++ b/apps/webapp/app/services/clickhouseInstance.server.ts @@ -12,29 +12,6 @@ function initializeClickhouseClient() { console.log(`🗃️ Clickhouse service enabled to host ${url.host}`); - // Build logs query settings from environment variables - const logsQuerySettings = { - list: { - max_memory_usage: env.CLICKHOUSE_LOGS_LIST_MAX_MEMORY_USAGE.toString(), - max_bytes_before_external_sort: - env.CLICKHOUSE_LOGS_LIST_MAX_BYTES_BEFORE_EXTERNAL_SORT.toString(), - max_threads: env.CLICKHOUSE_LOGS_LIST_MAX_THREADS, - ...(env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ && { - max_rows_to_read: env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ.toString(), - }), - ...(env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME && { - max_execution_time: env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME, - }), - }, - detail: { - max_memory_usage: env.CLICKHOUSE_LOGS_DETAIL_MAX_MEMORY_USAGE.toString(), - max_threads: env.CLICKHOUSE_LOGS_DETAIL_MAX_THREADS, - ...(env.CLICKHOUSE_LOGS_DETAIL_MAX_EXECUTION_TIME && { - max_execution_time: env.CLICKHOUSE_LOGS_DETAIL_MAX_EXECUTION_TIME, - }), - }, - }; - const clickhouse = new ClickHouse({ url: url.toString(), name: "clickhouse-instance", @@ -47,12 +24,53 @@ function initializeClickhouseClient() { request: true, }, maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, - logsQuerySettings, }); return clickhouse; } +export const logsClickhouseClient = singleton( + "logsClickhouseClient", + initializeLogsClickhouseClient +); + +function initializeLogsClickhouseClient() { + if (!env.LOGS_CLICKHOUSE_URL) { + throw new Error("LOGS_CLICKHOUSE_URL is not set"); + } + + const url = new URL(env.LOGS_CLICKHOUSE_URL); + + // Remove secure param + url.searchParams.delete("secure"); + + return new ClickHouse({ + url: url.toString(), + name: "logs-clickhouse", + keepAlive: { + enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", + idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, + }, + logLevel: env.CLICKHOUSE_LOG_LEVEL, + compression: { + request: true, + }, + maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, + clickhouseSettings: { + max_memory_usage: env.CLICKHOUSE_LOGS_LIST_MAX_MEMORY_USAGE.toString(), + max_bytes_before_external_sort: + env.CLICKHOUSE_LOGS_LIST_MAX_BYTES_BEFORE_EXTERNAL_SORT.toString(), + max_threads: env.CLICKHOUSE_LOGS_LIST_MAX_THREADS, + ...(env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ && { + max_rows_to_read: env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ.toString(), + }), + ...(env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME && { + max_execution_time: env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME, + }), + }, + }); +} + export const queryClickhouseClient = singleton( "queryClickhouseClient", initializeQueryClickhouseClient diff --git a/apps/webapp/app/utils/logUtils.ts b/apps/webapp/app/utils/logUtils.ts index cad9bbc9070..71ec44534b0 100644 --- a/apps/webapp/app/utils/logUtils.ts +++ b/apps/webapp/app/utils/logUtils.ts @@ -1,10 +1,10 @@ import { createElement, Fragment, type ReactNode } from "react"; import { z } from "zod"; -export const LogLevelSchema = z.enum(["DEBUG", "INFO", "WARN", "ERROR"]); +export const LogLevelSchema = z.enum(["TRACE", "DEBUG", "INFO", "WARN", "ERROR"]); export type LogLevel = z.infer; -export const validLogLevels: LogLevel[] = ["DEBUG", "INFO", "WARN", "ERROR",]; +export const validLogLevels: LogLevel[] = ["TRACE", "DEBUG", "INFO", "WARN", "ERROR"]; // Default styles for search highlighting const DEFAULT_HIGHLIGHT_STYLES: React.CSSProperties = { @@ -87,6 +87,7 @@ export function kindToLevel(kind: string, status: string): LogLevel { case "LOG_LOG": return "INFO"; // Changed from "LOG" case "SPAN": + return "TRACE"; case "ANCESTOR_OVERRIDE": case "SPAN_EVENT": default: @@ -101,6 +102,8 @@ export function getLevelColor(level: LogLevel): string { return "text-error bg-error/10 border-error/20"; case "WARN": return "text-warning bg-warning/10 border-warning/20"; + case "TRACE": + return "text-purple-400 bg-purple-500/10 border-purple-500/20"; case "DEBUG": return "text-charcoal-400 bg-charcoal-700 border-charcoal-600"; case "INFO": diff --git a/apps/webapp/tailwind.config.js b/apps/webapp/tailwind.config.js index f17eb04c15b..b3d6e2b235f 100644 --- a/apps/webapp/tailwind.config.js +++ b/apps/webapp/tailwind.config.js @@ -166,7 +166,7 @@ const deployments = colors.green[500]; const concurrency = colors.amber[500]; const limits = colors.purple[500]; const regions = colors.green[500]; -const logs = colors.blue[500]; +const logs = colors.pink[500]; const tests = colors.lime[500]; const apiKeys = colors.amber[500]; const environmentVariables = colors.pink[500]; diff --git a/internal-packages/clickhouse/schema/017_update_materialized_conditions_to_task_events_search_v1.sql b/internal-packages/clickhouse/schema/017_update_materialized_conditions_to_task_events_search_v1.sql new file mode 100644 index 00000000000..3f84cf555f2 --- /dev/null +++ b/internal-packages/clickhouse/schema/017_update_materialized_conditions_to_task_events_search_v1.sql @@ -0,0 +1,62 @@ +-- +goose Up +-- We drop the existing MV and recreate it with the new filter condition +DROP VIEW IF EXISTS trigger_dev.task_events_search_mv_v1; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.task_events_search_mv_v1 +TO trigger_dev.task_events_search_v1 AS +SELECT + environment_id, + organization_id, + project_id, + trace_id, + span_id, + run_id, + task_identifier, + start_time, + inserted_at, + message, + kind, + status, + duration, + parent_span_id, + attributes_text, + fromUnixTimestamp64Nano(toUnixTimestamp64Nano(start_time) + toInt64(duration)) AS triggered_timestamp +FROM trigger_dev.task_events_v2 +WHERE + trace_id != '' -- New condition added here + AND kind != 'DEBUG_EVENT' + AND status != 'PARTIAL' + AND NOT (kind = 'SPAN_EVENT' AND attributes_text = '{}') + AND kind != 'ANCESTOR_OVERRIDE' + AND message != 'trigger.dev/start'; + +-- +goose Down +-- In the down migration, we revert to the previous filter set +DROP VIEW IF EXISTS trigger_dev.task_events_search_mv_v1; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trigger_dev.task_events_search_mv_v1 +TO trigger_dev.task_events_search_v1 AS +SELECT + environment_id, + organization_id, + project_id, + trace_id, + span_id, + run_id, + task_identifier, + start_time, + inserted_at, + message, + kind, + status, + duration, + parent_span_id, + attributes_text, + fromUnixTimestamp64Nano(toUnixTimestamp64Nano(start_time) + toInt64(duration)) AS triggered_timestamp +FROM trigger_dev.task_events_v2 +WHERE + kind != 'DEBUG_EVENT' + AND status != 'PARTIAL' + AND NOT (kind = 'SPAN_EVENT' AND attributes_text = '{}') + AND kind != 'ANCESTOR_OVERRIDE' + AND message != 'trigger.dev/start'; \ No newline at end of file diff --git a/internal-packages/clickhouse/schema/018_drop_unused_task_events_v2_indexes.sql b/internal-packages/clickhouse/schema/018_drop_unused_task_events_v2_indexes.sql new file mode 100644 index 00000000000..81e3dbab61c --- /dev/null +++ b/internal-packages/clickhouse/schema/018_drop_unused_task_events_v2_indexes.sql @@ -0,0 +1,22 @@ +-- +goose Up + +-- These indexes are not used in any WHERE clause. +-- idx_duration: duration is only written/read as a column, never filtered on. +-- idx_attributes_text: search queries use the task_events_search_v1 table instead. +ALTER TABLE trigger_dev.task_events_v2 + DROP INDEX IF EXISTS idx_duration; + +ALTER TABLE trigger_dev.task_events_v2 + DROP INDEX IF EXISTS idx_attributes_text; + +-- +goose Down + +ALTER TABLE trigger_dev.task_events_v2 + ADD INDEX IF NOT EXISTS idx_duration duration + TYPE minmax + GRANULARITY 1; + +ALTER TABLE trigger_dev.task_events_v2 + ADD INDEX IF NOT EXISTS idx_attributes_text attributes_text + TYPE tokenbf_v1(32768, 3, 0) + GRANULARITY 8; diff --git a/internal-packages/clickhouse/src/client/queryBuilder.ts b/internal-packages/clickhouse/src/client/queryBuilder.ts index 30aad98486c..e802fc11bf3 100644 --- a/internal-packages/clickhouse/src/client/queryBuilder.ts +++ b/internal-packages/clickhouse/src/client/queryBuilder.ts @@ -4,6 +4,11 @@ import { ClickHouseSettings } from "@clickhouse/client"; export type QueryParamValue = string | number | boolean | Array | null; export type QueryParams = Record; +export type WhereCondition = { + clause: string; + params?: QueryParams; +}; + export class ClickhouseQueryBuilder { private name: string; private baseQuery: string; @@ -45,6 +50,20 @@ export class ClickhouseQueryBuilder { return this; } + whereOr(conditions: WhereCondition[]): this { + if (conditions.length === 0) { + return this; + } + const combinedClause = conditions.map((c) => `(${c.clause})`).join(" OR "); + this.whereClauses.push(`(${combinedClause})`); + for (const condition of conditions) { + if (condition.params) { + Object.assign(this.params, condition.params); + } + } + return this; + } + groupBy(clause: string): this { this.groupByClause = clause; return this; @@ -153,6 +172,20 @@ export class ClickhouseQueryFastBuilder> { return this; } + whereOr(conditions: WhereCondition[]): this { + if (conditions.length === 0) { + return this; + } + const combinedClause = conditions.map((c) => `(${c.clause})`).join(" OR "); + this.whereClauses.push(`(${combinedClause})`); + for (const condition of conditions) { + if (condition.params) { + Object.assign(this.params, condition.params); + } + } + return this; + } + groupBy(clause: string): this { this.groupByClause = clause; return this; diff --git a/internal-packages/clickhouse/src/index.ts b/internal-packages/clickhouse/src/index.ts index 47c2f34f2f5..de6bbb44e88 100644 --- a/internal-packages/clickhouse/src/index.ts +++ b/internal-packages/clickhouse/src/index.ts @@ -61,11 +61,6 @@ export type { OutputColumnMetadata } from "@internal/tsql"; // Errors export { QueryError } from "./client/errors.js"; -export type LogsQuerySettings = { - list?: ClickHouseSettings; - detail?: ClickHouseSettings; -}; - export type ClickhouseCommonConfig = { keepAlive?: { enabled?: boolean; @@ -80,7 +75,6 @@ export type ClickhouseCommonConfig = { response?: boolean; }; maxOpenConnections?: number; - logsQuerySettings?: LogsQuerySettings; }; export type ClickHouseConfig = @@ -104,11 +98,9 @@ export class ClickHouse { public readonly writer: ClickhouseWriter; private readonly logger: Logger; private _splitClients: boolean; - private readonly logsQuerySettings?: LogsQuerySettings; constructor(config: ClickHouseConfig) { this.logger = config.logger ?? new Logger("ClickHouse", config.logLevel ?? "debug"); - this.logsQuerySettings = config.logsQuerySettings; if (config.url) { const url = new URL(config.url); @@ -220,13 +212,13 @@ export class ClickHouse { traceSummaryQueryBuilder: getTraceSummaryQueryBuilderV2(this.reader), traceDetailedSummaryQueryBuilder: getTraceDetailedSummaryQueryBuilderV2(this.reader), spanDetailsQueryBuilder: getSpanDetailsQueryBuilderV2(this.reader), - logDetailQueryBuilder: getLogDetailQueryBuilderV2(this.reader, this.logsQuerySettings?.detail), + logDetailQueryBuilder: getLogDetailQueryBuilderV2(this.reader), }; } get taskEventsSearch() { return { - logsListQueryBuilder: getLogsSearchListQueryBuilder(this.reader, this.logsQuerySettings?.list), + logsListQueryBuilder: getLogsSearchListQueryBuilder(this.reader), }; } } diff --git a/internal-packages/clickhouse/src/taskEvents.ts b/internal-packages/clickhouse/src/taskEvents.ts index 73e2d8344ed..2109c498013 100644 --- a/internal-packages/clickhouse/src/taskEvents.ts +++ b/internal-packages/clickhouse/src/taskEvents.ts @@ -256,10 +256,7 @@ export const LogsSearchListResult = z.object({ export type LogsSearchListResult = z.output; -export function getLogsSearchListQueryBuilder( - ch: ClickhouseReader, - settings?: ClickHouseSettings -) { +export function getLogsSearchListQueryBuilder(ch: ClickhouseReader) { return ch.queryBuilderFast({ name: "getLogsSearchList", table: "trigger_dev.task_events_search_v1", @@ -280,7 +277,6 @@ export function getLogsSearchListQueryBuilder( "attributes_text", "triggered_timestamp", ], - settings, }); } @@ -304,7 +300,7 @@ export const LogDetailV2Result = z.object({ export type LogDetailV2Result = z.output; -export function getLogDetailQueryBuilderV2(ch: ClickhouseReader, settings?: ClickHouseSettings) { +export function getLogDetailQueryBuilderV2(ch: ClickhouseReader) { return ch.queryBuilderFast({ name: "getLogDetail", table: "trigger_dev.task_events_v2", @@ -324,6 +320,5 @@ export function getLogDetailQueryBuilderV2(ch: ClickhouseReader, settings?: Clic "duration", "attributes_text", ], - settings, }); } \ No newline at end of file From 59b6eb9a3e843809ffd5dba37775edc6e852592f Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: 2026年2月18日 10:39:38 +0000 Subject: [PATCH 2/5] feat(webapp): add region selector to test and replay task (#3082) Adds a region selector to the Test task page and Replay run dialog, so users can override the region from the dashboard. Disabled with a placeholder for dev environments. Closes #3016 --- .../components/runs/v3/ReplayRunDialog.tsx | 30 ++++ .../route.tsx | 148 ++++++++++++++++-- .../resources.taskruns.$runParam.replay.ts | 20 ++- .../app/v3/services/replayTaskRun.server.ts | 2 +- .../webapp/app/v3/services/testTask.server.ts | 2 + apps/webapp/app/v3/testTask.ts | 1 + 6 files changed, 184 insertions(+), 19 deletions(-) diff --git a/apps/webapp/app/components/runs/v3/ReplayRunDialog.tsx b/apps/webapp/app/components/runs/v3/ReplayRunDialog.tsx index 9192020e1bc..e42a2122abe 100644 --- a/apps/webapp/app/components/runs/v3/ReplayRunDialog.tsx +++ b/apps/webapp/app/components/runs/v3/ReplayRunDialog.tsx @@ -201,6 +201,7 @@ function ReplayForm({ tags, version, machine, + region, prioritySeconds, }, ] = useForm({ @@ -357,6 +358,35 @@ function ReplayForm({ )} {version.error} + {replayData.regions.length> 1 && ( + + + + {replayData.disableVersionSelection ? ( + Region is not available in the development environment. + ) : ( + Overrides the region for this run. + )} + {region.error} + + )} + {regionItems.length> 1 && ( + + + {/* Our Select primitive uses Ariakit under the hood, which treats + value={undefined} as uncontrolled, keeping stale internal state when + switching environments. The key forces a remount so it reinitializes + with the correct defaultValue. */} + + {isDev ? ( + Region is not available in the development environment. + ) : ( + Overrides the region for this run. + )} + {region.error} + + )} + {regionItems.length> 1 && ( + + + {/* Our Select primitive uses Ariakit under the hood, which treats + value={undefined} as uncontrolled, keeping stale internal state when + switching environments. The key forces a remount so it reinitializes + with the correct defaultValue. */} + + {isDev ? ( + Region is not available in the development environment. + ) : ( + Overrides the region for this run. + )} + {region.error} + + )}