From 1a979967f9eae040d0cd8bad0dafa6a773b69db5 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月11日 11:25:32 +0930 Subject: [PATCH 01/43] feat: update domain --- hosting/docker/docker-compose.traefik.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/hosting/docker/docker-compose.traefik.yml b/hosting/docker/docker-compose.traefik.yml index 297c55d74cc..3f6ccfe7252 100644 --- a/hosting/docker/docker-compose.traefik.yml +++ b/hosting/docker/docker-compose.traefik.yml @@ -6,9 +6,9 @@ services: - traefik labels: - "traefik.enable=true" - - "traefik.http.routers.webapp.rule=Host(`webapp.localhost`)" + - "traefik.http.routers.webapp.rule=Host(`trigger.lattebit.com`)" - "traefik.http.routers.webapp.entrypoints=${TRAEFIK_ENTRYPOINT:-web}" - # - "traefik.http.routers.webapp.tls.certresolver=letsencrypt" + - "traefik.http.routers.webapp.tls.certresolver=letsencrypt" - "traefik.http.services.webapp.loadbalancer.server.port=3000" registry: @@ -16,9 +16,9 @@ services: - traefik labels: - "traefik.enable=true" - - "traefik.http.routers.registry.rule=Host(`registry.localhost`)" + - "traefik.http.routers.registry.rule=Host(`registry.lattebit.com`)" - "traefik.http.routers.registry.entrypoints=${TRAEFIK_ENTRYPOINT:-web}" - # - "traefik.http.routers.registry.tls.certresolver=letsencrypt" + - "traefik.http.routers.registry.tls.certresolver=letsencrypt" - "traefik.http.services.registry.loadbalancer.server.port=5000" minio: @@ -26,9 +26,9 @@ services: - traefik labels: - "traefik.enable=true" - - "traefik.http.routers.minio.rule=Host(`minio.localhost`)" + - "traefik.http.routers.minio.rule=Host(`minio.lattebit.com`)" - "traefik.http.routers.minio.entrypoints=${TRAEFIK_ENTRYPOINT:-web}" - # - "traefik.http.routers.minio.tls.certresolver=letsencrypt" + - "traefik.http.routers.minio.tls.certresolver=letsencrypt" - "traefik.http.services.minio.loadbalancer.server.port=9000" traefik: @@ -47,9 +47,9 @@ services: - --providers.docker.network=traefik - --entrypoints.web.address=:80 - --entrypoints.websecure.address=:443 - # - --certificatesresolvers.letsencrypt.acme.tlschallenge=true - # - --certificatesresolvers.letsencrypt.acme.email=local@example.com - # - --certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json + - --certificatesresolvers.letsencrypt.acme.tlschallenge=true + - --certificatesresolvers.letsencrypt.acme.email=dqaria@gmail.com + - --certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json - --log.level=DEBUG volumes: - /var/run/docker.sock:/var/run/docker.sock:ro From acc1a549ec5ff4025ed3c34580030074f467ebe1 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月11日 11:39:47 +0930 Subject: [PATCH 02/43] feat: add link --- hosting/docker/webapp/docker-compose.traefik.yml | 1 + 1 file changed, 1 insertion(+) create mode 120000 hosting/docker/webapp/docker-compose.traefik.yml diff --git a/hosting/docker/webapp/docker-compose.traefik.yml b/hosting/docker/webapp/docker-compose.traefik.yml new file mode 120000 index 00000000000..a962deee876 --- /dev/null +++ b/hosting/docker/webapp/docker-compose.traefik.yml @@ -0,0 +1 @@ +../docker-compose.traefik.yml \ No newline at end of file From 22d908613ba33b8fd4e56b75714d8b663dcd6e1f Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月11日 12:08:15 +0930 Subject: [PATCH 03/43] feat: add auth --- hosting/docker/webapp/docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index 1935ad5edcd..ef60d830599 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -79,6 +79,10 @@ services: INTERNAL_OTEL_TRACE_LOGGING_ENABLED: ${INTERNAL_OTEL_TRACE_LOGGING_ENABLED:-0} TRIGGER_CLI_TAG: ${TRIGGER_CLI_TAG:-v4-beta} + AUTH_GITHUB_CLIENT_ID: ${AUTH_GITHUB_CLIENT_ID} + AUTH_GITHUB_CLIENT_SECRET: ${AUTH_GITHUB_CLIENT_SECRET} + + postgres: image: postgres:${POSTGRES_IMAGE_TAG:-14} restart: ${RESTART_POLICY:-unless-stopped} From e128fbceec169b1f6750882e9672878b41ca4032 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月11日 12:40:52 +0930 Subject: [PATCH 04/43] fix: runner --- hosting/docker/worker/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/hosting/docker/worker/docker-compose.yml b/hosting/docker/worker/docker-compose.yml index 6e9f4db2721..12b339d164b 100644 --- a/hosting/docker/worker/docker-compose.yml +++ b/hosting/docker/worker/docker-compose.yml @@ -34,6 +34,7 @@ services: OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://webapp:3000/otel} TRIGGER_WORKLOAD_API_DOMAIN: supervisor TRIGGER_WORKLOAD_API_PORT_EXTERNAL: 8020 + TRIGGER_DEQUEUE_MAX_RUN_COUNT: ${TRIGGER_DEQUEUE_MAX_RUN_COUNT:-2} # Optional settings DEBUG: 1 ENFORCE_MACHINE_PRESETS: 1 From d10dca1a54001590d5e5957a179af4794012ce6e Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月11日 13:11:30 +0930 Subject: [PATCH 05/43] feat: add timeout & worker --- hosting/docker/docker-compose.traefik.yml | 30 +++++++++++++++++++++++ hosting/docker/worker/docker-compose.yml | 4 +-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/hosting/docker/docker-compose.traefik.yml b/hosting/docker/docker-compose.traefik.yml index 3f6ccfe7252..a43ee227084 100644 --- a/hosting/docker/docker-compose.traefik.yml +++ b/hosting/docker/docker-compose.traefik.yml @@ -20,6 +20,19 @@ services: - "traefik.http.routers.registry.entrypoints=${TRAEFIK_ENTRYPOINT:-web}" - "traefik.http.routers.registry.tls.certresolver=letsencrypt" - "traefik.http.services.registry.loadbalancer.server.port=5000" + # 增加超时设置 + - "traefik.http.services.registry.loadbalancer.responseForwarding.readTimeout=600s" + - "traefik.http.services.registry.loadbalancer.server.scheme=http" + + # 创建并应用 buffering 中间件(对大文件传输很重要) + - "traefik.http.middlewares.registry-buffer.buffering.memRequestBodyBytes=2147483648" # 2GB + - "traefik.http.middlewares.registry-buffer.buffering.maxRequestBodyBytes=0" # 不限制 + - "traefik.http.middlewares.registry-buffer.buffering.memResponseBodyBytes=2147483648" # 2GB + - "traefik.http.middlewares.registry-buffer.buffering.maxResponseBodyBytes=0" # 不限制 + - "traefik.http.middlewares.registry-buffer.buffering.retryExpression=IsNetworkError() && Attempts() < 2" + + # 应用中间件到路由 + - "traefik.http.routers.registry.middlewares=registry-buffer" minio: networks: @@ -51,6 +64,23 @@ services: - --certificatesresolvers.letsencrypt.acme.email=dqaria@gmail.com - --certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json - --log.level=DEBUG + + + # 添加全局超时设置 + - --serversTransport.forwardingTimeouts.dialTimeout=30s + - --serversTransport.forwardingTimeouts.responseHeaderTimeout=600s + - --serversTransport.forwardingTimeouts.idleConnTimeout=90s + - --serversTransport.insecureSkipVerify=true + - --serversTransport.maxIdleConnsPerHost=200 + + # 添加入口点的传输超时(可选,但建议添加) + - --entrypoints.web.transport.respondingTimeouts.readTimeout=600s + - --entrypoints.web.transport.respondingTimeouts.writeTimeout=600s + - --entrypoints.web.transport.respondingTimeouts.idleTimeout=180s + - --entrypoints.websecure.transport.respondingTimeouts.readTimeout=600s + - --entrypoints.websecure.transport.respondingTimeouts.writeTimeout=600s + - --entrypoints.websecure.transport.respondingTimeouts.idleTimeout=180s + volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - traefik-letsencrypt:/letsencrypt diff --git a/hosting/docker/worker/docker-compose.yml b/hosting/docker/worker/docker-compose.yml index 12b339d164b..43e645ad5c3 100644 --- a/hosting/docker/worker/docker-compose.yml +++ b/hosting/docker/worker/docker-compose.yml @@ -26,9 +26,9 @@ services: command: sh -c "chown -R node:node /home/node/shared && exec /usr/bin/dumb-init -- pnpm run --filter supervisor start" environment: # This needs to match the token of the worker group you want to connect to - # TRIGGER_WORKER_TOKEN: ${TRIGGER_WORKER_TOKEN} + TRIGGER_WORKER_TOKEN: ${TRIGGER_WORKER_TOKEN} # Use the bootstrap token created by the webapp - TRIGGER_WORKER_TOKEN: file:///home/node/shared/worker_token + # TRIGGER_WORKER_TOKEN: file:///home/node/shared/worker_token MANAGED_WORKER_SECRET: ${MANAGED_WORKER_SECRET} TRIGGER_API_URL: ${TRIGGER_API_URL:-http://webapp:3000} OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://webapp:3000/otel} From afec9d362268d77a1e50241810146c9d20969e6e Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月11日 13:53:59 +0930 Subject: [PATCH 06/43] fix --- hosting/docker/worker/docker-compose.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hosting/docker/worker/docker-compose.yml b/hosting/docker/worker/docker-compose.yml index 43e645ad5c3..d1bc85b8cb9 100644 --- a/hosting/docker/worker/docker-compose.yml +++ b/hosting/docker/worker/docker-compose.yml @@ -34,7 +34,8 @@ services: OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://webapp:3000/otel} TRIGGER_WORKLOAD_API_DOMAIN: supervisor TRIGGER_WORKLOAD_API_PORT_EXTERNAL: 8020 - TRIGGER_DEQUEUE_MAX_RUN_COUNT: ${TRIGGER_DEQUEUE_MAX_RUN_COUNT:-2} + TRIGGER_DEQUEUE_MAX_RUN_COUNT: 2 + TRIGGER_DEQUEUE_MAX_CONSUMER_COUNT: 1 # Optional settings DEBUG: 1 ENFORCE_MACHINE_PRESETS: 1 @@ -42,8 +43,8 @@ services: DOCKER_HOST: tcp://docker-proxy:2375 DOCKER_RUNNER_NETWORKS: webapp,supervisor DOCKER_REGISTRY_URL: ${DOCKER_REGISTRY_URL:-localhost:5000} - DOCKER_REGISTRY_USERNAME: ${DOCKER_REGISTRY_USERNAME:-} - DOCKER_REGISTRY_PASSWORD: ${DOCKER_REGISTRY_PASSWORD:-} + # DOCKER_REGISTRY_USERNAME: ${DOCKER_REGISTRY_USERNAME:-} + # DOCKER_REGISTRY_PASSWORD: ${DOCKER_REGISTRY_PASSWORD:-} DOCKER_AUTOREMOVE_EXITED_CONTAINERS: 0 healthcheck: test: ["CMD", "node", "-e", "http.get('http://localhost:8020/health', res => process.exit(res.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] From 659ceccb5eb4331a9d572167fe5045c728dffd1b Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月11日 14:33:00 +0930 Subject: [PATCH 07/43] feat: add max run count limit --- apps/supervisor/src/env.ts | 2 ++ apps/supervisor/src/index.ts | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index fd6bd610509..3b2a55bcb1f 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -38,6 +38,8 @@ const Env = z.object({ TRIGGER_DEQUEUE_MAX_RUN_COUNT: z.coerce.number().int().default(10), TRIGGER_DEQUEUE_MAX_CONSUMER_COUNT: z.coerce.number().int().default(1), + TRIGGER_WORKER_MAX_RUN_COUNT: z.coerce.number().int().default(4), + // Optional services TRIGGER_WARM_START_URL: z.string().optional(), TRIGGER_CHECKPOINT_URL: z.string().optional(), diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts index 83fe89c1ed0..1b0b473e8f7 100644 --- a/apps/supervisor/src/index.ts +++ b/apps/supervisor/src/index.ts @@ -31,6 +31,7 @@ if (env.METRICS_COLLECT_DEFAULTS) { } class ManagedSupervisor { + private activeTaskCount = 0; private readonly workerSession: SupervisorSession; private readonly metricsServer?: HttpServer; private readonly workloadServer: WorkloadServer; @@ -142,6 +143,10 @@ class ManagedSupervisor { return {}; } + if (this.activeTaskCount>= env.TRIGGER_WORKER_MAX_RUN_COUNT) { //限制最多运行的任务 + return { skipDequeue: true }; + } + const resources = await this.resourceMonitor.getNodeResources(); return { @@ -177,6 +182,7 @@ class ManagedSupervisor { }); this.workerSession.on("runQueueMessage", async ({ time, message }) => { + this.activeTaskCount++; this.logger.log(`Received message with timestamp ${time.toLocaleString()}`, message); if (message.completedWaitpoints.length> 0) { @@ -288,6 +294,7 @@ class ManagedSupervisor { } async onRunDisconnected({ run }: { run: { friendlyId: string } }) { + this.activeTaskCount = Math.max(0, this.activeTaskCount - 1); this.logger.debug("Run disconnected", { run }); this.workerSession.unsubscribeFromRunNotifications([run.friendlyId]); } From 1863764e3a2f2d40f720557e1b20a9a3841c8704 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月11日 14:44:50 +0930 Subject: [PATCH 08/43] use custom build --- hosting/docker/worker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/docker/worker/docker-compose.yml b/hosting/docker/worker/docker-compose.yml index d1bc85b8cb9..e123c8e5746 100644 --- a/hosting/docker/worker/docker-compose.yml +++ b/hosting/docker/worker/docker-compose.yml @@ -9,7 +9,7 @@ x-logging: &logging-config services: supervisor: - image: ghcr.io/triggerdotdev/supervisor:${TRIGGER_IMAGE_TAG:-v4-beta} + image: ghcr.io/lattebit/supervisor:latest restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config depends_on: From e5341eee59cf9fb1e933c7cd88464ebfc5954690 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月11日 14:53:28 +0930 Subject: [PATCH 09/43] feat: add limit --- apps/supervisor/src/index.ts | 42 +++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts index 1b0b473e8f7..2f4aab60db7 100644 --- a/apps/supervisor/src/index.ts +++ b/apps/supervisor/src/index.ts @@ -134,6 +134,15 @@ class ManagedSupervisor { heartbeatIntervalSeconds: env.TRIGGER_WORKER_HEARTBEAT_INTERVAL_SECONDS, sendRunDebugLogs: env.SEND_RUN_DEBUG_LOGS, preDequeue: async () => { + // 任务数量限制检查(优先级最高) + if (this.activeTaskCount>= env.TRIGGER_WORKER_MAX_RUN_COUNT) { + this.logger.debug("Skipping dequeue due to activeTaskCount limit", { + activeTaskCount: this.activeTaskCount, + maxRunCount: env.TRIGGER_WORKER_MAX_RUN_COUNT, + }); + return { skipDequeue: true }; + } + if (!env.RESOURCE_MONITOR_ENABLED) { return {}; } @@ -143,18 +152,22 @@ class ManagedSupervisor { return {}; } - if (this.activeTaskCount>= env.TRIGGER_WORKER_MAX_RUN_COUNT) { //限制最多运行的任务 - return { skipDequeue: true }; - } - const resources = await this.resourceMonitor.getNodeResources(); + const shouldSkip = resources.cpuAvailable < 0.25 || resources.memoryAvailable < 0.25; + if (shouldSkip) { + this.logger.debug("Skipping dequeue due to resource constraints", { + cpuAvailable: resources.cpuAvailable, + memoryAvailable: resources.memoryAvailable, + }); + } + return { maxResources: { cpu: resources.cpuAvailable, memory: resources.memoryAvailable, }, - skipDequeue: resources.cpuAvailable < 0.25 || resources.memoryAvailable < 0.25, + skipDequeue: shouldSkip, }; }, preSkip: async () => { @@ -182,8 +195,8 @@ class ManagedSupervisor { }); this.workerSession.on("runQueueMessage", async ({ time, message }) => { - this.activeTaskCount++; this.logger.log(`Received message with timestamp ${time.toLocaleString()}`, message); + this.logger.log(`Active tasks before processing: ${this.activeTaskCount}/${env.TRIGGER_WORKER_MAX_RUN_COUNT}`); if (message.completedWaitpoints.length> 0) { this.logger.debug("Run has completed waitpoints", { @@ -235,6 +248,8 @@ class ManagedSupervisor { if (didWarmStart) { this.logger.log("Warm start successful", { runId: message.run.id }); + this.activeTaskCount++; + this.logger.log(`Active tasks after warm start: ${this.activeTaskCount}/${env.TRIGGER_WORKER_MAX_RUN_COUNT}`); return; } @@ -255,6 +270,9 @@ class ManagedSupervisor { snapshotFriendlyId: message.snapshot.friendlyId, }); + this.activeTaskCount++; + this.logger.log(`Active tasks after creating workload: ${this.activeTaskCount}/${env.TRIGGER_WORKER_MAX_RUN_COUNT}`); + // Disabled for now // this.resourceMonitor.blockResources({ // cpu: message.run.machine.cpu, @@ -289,13 +307,21 @@ class ManagedSupervisor { } async onRunConnected({ run }: { run: { friendlyId: string } }) { - this.logger.debug("Run connected", { run }); + this.logger.debug("Run connected", { + run, + activeTaskCount: this.activeTaskCount, + maxRunCount: env.TRIGGER_WORKER_MAX_RUN_COUNT + }); this.workerSession.subscribeToRunNotifications([run.friendlyId]); } async onRunDisconnected({ run }: { run: { friendlyId: string } }) { this.activeTaskCount = Math.max(0, this.activeTaskCount - 1); - this.logger.debug("Run disconnected", { run }); + this.logger.debug("Run disconnected", { + run, + activeTaskCount: this.activeTaskCount, + maxRunCount: env.TRIGGER_WORKER_MAX_RUN_COUNT + }); this.workerSession.unsubscribeFromRunNotifications([run.friendlyId]); } From 9bade49ab6b9bec19a52257039d091b849dab8b3 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月11日 14:59:47 +0930 Subject: [PATCH 10/43] feat: add limit --- hosting/docker/worker/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hosting/docker/worker/docker-compose.yml b/hosting/docker/worker/docker-compose.yml index e123c8e5746..32fb8566e6f 100644 --- a/hosting/docker/worker/docker-compose.yml +++ b/hosting/docker/worker/docker-compose.yml @@ -34,8 +34,8 @@ services: OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://webapp:3000/otel} TRIGGER_WORKLOAD_API_DOMAIN: supervisor TRIGGER_WORKLOAD_API_PORT_EXTERNAL: 8020 - TRIGGER_DEQUEUE_MAX_RUN_COUNT: 2 - TRIGGER_DEQUEUE_MAX_CONSUMER_COUNT: 1 + RESOURCE_MONITOR_ENABLED: 1 + TRIGGER_WORKER_MAX_RUN_COUNT: ${TRIGGER_WORKER_MAX_RUN_COUNT:-4} # Optional settings DEBUG: 1 ENFORCE_MACHINE_PRESETS: 1 From 2a4a3c838fac7a639f0cca9043e9b6039f5b647d Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月11日 15:12:11 +0930 Subject: [PATCH 11/43] update --- hosting/docker/worker/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hosting/docker/worker/docker-compose.yml b/hosting/docker/worker/docker-compose.yml index 32fb8566e6f..0d924916b39 100644 --- a/hosting/docker/worker/docker-compose.yml +++ b/hosting/docker/worker/docker-compose.yml @@ -34,8 +34,8 @@ services: OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://webapp:3000/otel} TRIGGER_WORKLOAD_API_DOMAIN: supervisor TRIGGER_WORKLOAD_API_PORT_EXTERNAL: 8020 - RESOURCE_MONITOR_ENABLED: 1 - TRIGGER_WORKER_MAX_RUN_COUNT: ${TRIGGER_WORKER_MAX_RUN_COUNT:-4} + #RESOURCE_MONITOR_ENABLED: 1 + TRIGGER_WORKER_MAX_RUN_COUNT: ${TRIGGER_WORKER_MAX_RUN_COUNT:-2} # Optional settings DEBUG: 1 ENFORCE_MACHINE_PRESETS: 1 From eca3c24aafd2bd38bda00e2b9a793088dfda59f3 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月12日 16:21:51 +0930 Subject: [PATCH 12/43] feat: use supabase --- hosting/docker/docker-compose.traefik.yml | 51 +--------------------- hosting/docker/webapp/docker-compose.yml | 53 +---------------------- 2 files changed, 2 insertions(+), 102 deletions(-) diff --git a/hosting/docker/docker-compose.traefik.yml b/hosting/docker/docker-compose.traefik.yml index a43ee227084..246c151a568 100644 --- a/hosting/docker/docker-compose.traefik.yml +++ b/hosting/docker/docker-compose.traefik.yml @@ -10,40 +10,7 @@ services: - "traefik.http.routers.webapp.entrypoints=${TRAEFIK_ENTRYPOINT:-web}" - "traefik.http.routers.webapp.tls.certresolver=letsencrypt" - "traefik.http.services.webapp.loadbalancer.server.port=3000" - - registry: - networks: - - traefik - labels: - - "traefik.enable=true" - - "traefik.http.routers.registry.rule=Host(`registry.lattebit.com`)" - - "traefik.http.routers.registry.entrypoints=${TRAEFIK_ENTRYPOINT:-web}" - - "traefik.http.routers.registry.tls.certresolver=letsencrypt" - - "traefik.http.services.registry.loadbalancer.server.port=5000" - # 增加超时设置 - - "traefik.http.services.registry.loadbalancer.responseForwarding.readTimeout=600s" - - "traefik.http.services.registry.loadbalancer.server.scheme=http" - - # 创建并应用 buffering 中间件(对大文件传输很重要) - - "traefik.http.middlewares.registry-buffer.buffering.memRequestBodyBytes=2147483648" # 2GB - - "traefik.http.middlewares.registry-buffer.buffering.maxRequestBodyBytes=0" # 不限制 - - "traefik.http.middlewares.registry-buffer.buffering.memResponseBodyBytes=2147483648" # 2GB - - "traefik.http.middlewares.registry-buffer.buffering.maxResponseBodyBytes=0" # 不限制 - - "traefik.http.middlewares.registry-buffer.buffering.retryExpression=IsNetworkError() && Attempts() < 2" - - # 应用中间件到路由 - - "traefik.http.routers.registry.middlewares=registry-buffer" - - minio: - networks: - - traefik - labels: - - "traefik.enable=true" - - "traefik.http.routers.minio.rule=Host(`minio.lattebit.com`)" - - "traefik.http.routers.minio.entrypoints=${TRAEFIK_ENTRYPOINT:-web}" - - "traefik.http.routers.minio.tls.certresolver=letsencrypt" - - "traefik.http.services.minio.loadbalancer.server.port=9000" - + traefik: image: traefik:${TRAEFIK_IMAGE_TAG:-v3.4} restart: ${RESTART_POLICY:-unless-stopped} @@ -65,22 +32,6 @@ services: - --certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json - --log.level=DEBUG - - # 添加全局超时设置 - - --serversTransport.forwardingTimeouts.dialTimeout=30s - - --serversTransport.forwardingTimeouts.responseHeaderTimeout=600s - - --serversTransport.forwardingTimeouts.idleConnTimeout=90s - - --serversTransport.insecureSkipVerify=true - - --serversTransport.maxIdleConnsPerHost=200 - - # 添加入口点的传输超时(可选,但建议添加) - - --entrypoints.web.transport.respondingTimeouts.readTimeout=600s - - --entrypoints.web.transport.respondingTimeouts.writeTimeout=600s - - --entrypoints.web.transport.respondingTimeouts.idleTimeout=180s - - --entrypoints.websecure.transport.respondingTimeouts.readTimeout=600s - - --entrypoints.websecure.transport.respondingTimeouts.writeTimeout=600s - - --entrypoints.websecure.transport.respondingTimeouts.idleTimeout=180s - volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - traefik-letsencrypt:/letsencrypt diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index ef60d830599..778ece97591 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -20,7 +20,6 @@ services: - clickhouse networks: - webapp - - supervisor volumes: - shared:/home/node/shared # Only needed for bootstrap @@ -70,7 +69,7 @@ services: RUN_REPLICATION_LOG_LEVEL: ${RUN_REPLICATION_LOG_LEVEL:-info} # Limits # TASK_PAYLOAD_OFFLOAD_THRESHOLD: 524288 # 512KB - # TASK_PAYLOAD_MAXIMUM_SIZE: 3145728 # 3MB + TASK_PAYLOAD_MAXIMUM_SIZE: 31457280 # 30MB # BATCH_TASK_PAYLOAD_MAXIMUM_SIZE: 1000000 # 1MB # TASK_RUN_METADATA_MAXIMUM_SIZE: 262144 # 256KB # DEFAULT_ENV_EXECUTION_CONCURRENCY_LIMIT: 100 @@ -82,31 +81,6 @@ services: AUTH_GITHUB_CLIENT_ID: ${AUTH_GITHUB_CLIENT_ID} AUTH_GITHUB_CLIENT_SECRET: ${AUTH_GITHUB_CLIENT_SECRET} - - postgres: - image: postgres:${POSTGRES_IMAGE_TAG:-14} - restart: ${RESTART_POLICY:-unless-stopped} - logging: *logging-config - ports: - - ${POSTGRES_PUBLISH_IP:-127.0.0.1}:5433:5432 - volumes: - - postgres:/var/lib/postgresql/data/ - networks: - - webapp - command: - - -c - - wal_level=logical - environment: - POSTGRES_USER: ${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} - POSTGRES_DB: ${POSTGRES_DB:-postgres} - healthcheck: - test: ["CMD", "pg_isready", "-U", "postgres"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 10s - redis: image: redis:${REDIS_IMAGE_TAG:-7} restart: ${RESTART_POLICY:-unless-stopped} @@ -165,28 +139,6 @@ services: retries: 5 start_period: 10s - registry: - image: registry:${REGISTRY_IMAGE_TAG:-2} - restart: ${RESTART_POLICY:-unless-stopped} - logging: *logging-config - ports: - - ${REGISTRY_PUBLISH_IP:-127.0.0.1}:5000:5000 - networks: - - webapp - volumes: - # registry-user:very-secure-indeed - - ../registry/auth.htpasswd:/auth/htpasswd:ro - environment: - REGISTRY_AUTH: htpasswd - REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm - REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd - healthcheck: - test: ["CMD", "wget", "--spider", "-q", "http://localhost:5000/"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 10s - minio: image: bitnami/minio:${MINIO_IMAGE_TAG:-latest} restart: ${RESTART_POLICY:-unless-stopped} @@ -212,7 +164,6 @@ services: volumes: clickhouse: - postgres: redis: shared: minio: @@ -220,7 +171,5 @@ volumes: networks: docker-proxy: name: docker-proxy - supervisor: - name: supervisor webapp: name: webapp \ No newline at end of file From 1e5efdfc16a0a72d35aabdad8879d62fab92f5ea Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月12日 16:28:42 +0930 Subject: [PATCH 13/43] rm dep --- hosting/docker/webapp/docker-compose.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index 778ece97591..bfa7f7e6e5f 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -15,7 +15,6 @@ services: ports: - ${WEBAPP_PUBLISH_IP:-0.0.0.0}:8030:3000 depends_on: - - postgres - redis - clickhouse networks: @@ -102,8 +101,6 @@ services: image: electricsql/electric:${ELECTRIC_IMAGE_TAG:-1.0.24} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config - depends_on: - - postgres networks: - webapp environment: From 998c4eb4edc45a84b6587884e370133bebec7ab4 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月12日 17:07:31 +0930 Subject: [PATCH 14/43] feat: use r2 to store files --- hosting/docker/webapp/docker-compose.yml | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index bfa7f7e6e5f..fe3a03bcbd5 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -136,34 +136,10 @@ services: retries: 5 start_period: 10s - minio: - image: bitnami/minio:${MINIO_IMAGE_TAG:-latest} - restart: ${RESTART_POLICY:-unless-stopped} - logging: *logging-config - ports: - - ${MINIO_PUBLISH_IP:-127.0.0.1}:9000:9000 - - ${MINIO_PUBLISH_IP:-127.0.0.1}:9001:9001 - networks: - - webapp - volumes: - - minio:/bitnami/minio/data - environment: - MINIO_ROOT_USER: ${MINIO_ROOT_USER:-admin} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-very-safe-password} - MINIO_DEFAULT_BUCKETS: packets - MINIO_BROWSER: "on" - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 5s - timeout: 10s - retries: 5 - start_period: 10s - volumes: clickhouse: redis: shared: - minio: networks: docker-proxy: From f425bd6544ace21fc03085e850eecc2459489c71 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月12日 17:35:27 +0930 Subject: [PATCH 15/43] feat: add ipv6 --- hosting/docker/webapp/docker-compose.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index fe3a03bcbd5..2ac6b18801d 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -25,6 +25,8 @@ services: user: root # Only needed for bootstrap command: sh -c "chown -R node:node /home/node/shared && exec ./scripts/entrypoint.sh" + sysctls: # ipv6 + - net.ipv6.conf.all.disable_ipv6=0 healthcheck: test: ["CMD", "node", "-e", "http.get('http://localhost:3000/healthcheck', res => process.exit(res.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] interval: 30s @@ -145,4 +147,8 @@ networks: docker-proxy: name: docker-proxy webapp: - name: webapp \ No newline at end of file + name: webapp + enable_ipv6: true + ipam: + config: + - subnet: fd00:1::/80 From e36afe36d7b8b843a643a65dee21e4e9388e4b65 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月12日 17:48:29 +0930 Subject: [PATCH 16/43] fix: change back --- hosting/docker/webapp/docker-compose.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index 2ac6b18801d..fe3a03bcbd5 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -25,8 +25,6 @@ services: user: root # Only needed for bootstrap command: sh -c "chown -R node:node /home/node/shared && exec ./scripts/entrypoint.sh" - sysctls: # ipv6 - - net.ipv6.conf.all.disable_ipv6=0 healthcheck: test: ["CMD", "node", "-e", "http.get('http://localhost:3000/healthcheck', res => process.exit(res.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] interval: 30s @@ -147,8 +145,4 @@ networks: docker-proxy: name: docker-proxy webapp: - name: webapp - enable_ipv6: true - ipam: - config: - - subnet: fd00:1::/80 + name: webapp \ No newline at end of file From 46cf922d9a91bf82be98d5b86df0565122e43737 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月12日 21:29:02 +0930 Subject: [PATCH 17/43] fix: add back local db --- hosting/docker/webapp/docker-compose.yml | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index fe3a03bcbd5..eeb9c201107 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -15,6 +15,7 @@ services: ports: - ${WEBAPP_PUBLISH_IP:-0.0.0.0}:8030:3000 depends_on: + - postgres - redis - clickhouse networks: @@ -80,6 +81,31 @@ services: AUTH_GITHUB_CLIENT_ID: ${AUTH_GITHUB_CLIENT_ID} AUTH_GITHUB_CLIENT_SECRET: ${AUTH_GITHUB_CLIENT_SECRET} + + postgres: + image: postgres:${POSTGRES_IMAGE_TAG:-14} + restart: ${RESTART_POLICY:-unless-stopped} + logging: *logging-config + ports: + - ${POSTGRES_PUBLISH_IP:-127.0.0.1}:5433:5432 + volumes: + - postgres:/var/lib/postgresql/data/ + networks: + - webapp + command: + - -c + - wal_level=logical + environment: + POSTGRES_USER: ${POSTGRES_USER:-postgres} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} + POSTGRES_DB: ${POSTGRES_DB:-postgres} + healthcheck: + test: ["CMD", "pg_isready", "-U", "postgres"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + redis: image: redis:${REDIS_IMAGE_TAG:-7} restart: ${RESTART_POLICY:-unless-stopped} @@ -101,6 +127,8 @@ services: image: electricsql/electric:${ELECTRIC_IMAGE_TAG:-1.0.24} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config + depends_on: + - postgres networks: - webapp environment: @@ -138,6 +166,7 @@ services: volumes: clickhouse: + postgres: redis: shared: From eb3e6a78b4a4e3343e2b4d1e605b515a276c0283 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月13日 16:32:08 +0930 Subject: [PATCH 18/43] feat: add dokploy support --- hosting/docker/webapp/docker-compose.yml | 48 +----------------------- 1 file changed, 2 insertions(+), 46 deletions(-) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index eeb9c201107..a211c6672d6 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -13,13 +13,10 @@ services: restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config ports: - - ${WEBAPP_PUBLISH_IP:-0.0.0.0}:8030:3000 + - "3000:3000" depends_on: - - postgres - redis - clickhouse - networks: - - webapp volumes: - shared:/home/node/shared # Only needed for bootstrap @@ -81,31 +78,6 @@ services: AUTH_GITHUB_CLIENT_ID: ${AUTH_GITHUB_CLIENT_ID} AUTH_GITHUB_CLIENT_SECRET: ${AUTH_GITHUB_CLIENT_SECRET} - - postgres: - image: postgres:${POSTGRES_IMAGE_TAG:-14} - restart: ${RESTART_POLICY:-unless-stopped} - logging: *logging-config - ports: - - ${POSTGRES_PUBLISH_IP:-127.0.0.1}:5433:5432 - volumes: - - postgres:/var/lib/postgresql/data/ - networks: - - webapp - command: - - -c - - wal_level=logical - environment: - POSTGRES_USER: ${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} - POSTGRES_DB: ${POSTGRES_DB:-postgres} - healthcheck: - test: ["CMD", "pg_isready", "-U", "postgres"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 10s - redis: image: redis:${REDIS_IMAGE_TAG:-7} restart: ${RESTART_POLICY:-unless-stopped} @@ -114,8 +86,6 @@ services: - ${REDIS_PUBLISH_IP:-127.0.0.1}:6389:6379 volumes: - redis:/data - networks: - - webapp healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 10s @@ -127,10 +97,6 @@ services: image: electricsql/electric:${ELECTRIC_IMAGE_TAG:-1.0.24} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config - depends_on: - - postgres - networks: - - webapp environment: DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/main?schema=public&sslmode=disable} ELECTRIC_INSECURE: true @@ -146,9 +112,6 @@ services: image: bitnami/clickhouse:${CLICKHOUSE_IMAGE_TAG:-latest} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config - ports: - - ${CLICKHOUSE_PUBLISH_IP:-127.0.0.1}:9123:8123 - - ${CLICKHOUSE_PUBLISH_IP:-127.0.0.1}:9090:9000 environment: CLICKHOUSE_ADMIN_USER: ${CLICKHOUSE_USER:-default} CLICKHOUSE_ADMIN_PASSWORD: ${CLICKHOUSE_PASSWORD:-password} @@ -166,12 +129,5 @@ services: volumes: clickhouse: - postgres: redis: - shared: - -networks: - docker-proxy: - name: docker-proxy - webapp: - name: webapp \ No newline at end of file + shared: \ No newline at end of file From 681114e8777682e528cc21855a21a2345aa91e69 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月13日 16:36:56 +0930 Subject: [PATCH 19/43] feat: clean network --- hosting/docker/webapp/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index a211c6672d6..ba95f575115 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -118,8 +118,6 @@ services: volumes: - clickhouse:/bitnami/clickhouse - ../clickhouse/override.xml:/bitnami/clickhouse/etc/config.d/override.xml:ro - networks: - - webapp healthcheck: test: ["CMD", "clickhouse-client", "--host", "localhost", "--port", "9000", "--user", "default", "--password", "password", "--query", "SELECT 1"] interval: 5s From 870299b682ab26bfb3fd8841bfa47c3d0010e044 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月13日 16:43:29 +0930 Subject: [PATCH 20/43] fix: port --- hosting/docker/webapp/docker-compose.yml | 4 ++-- hosting/docker/worker/docker-compose.yml | 16 +--------------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index ba95f575115..b0ec90b2ac0 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -13,7 +13,7 @@ services: restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config ports: - - "3000:3000" + - "8030:3000" depends_on: - redis - clickhouse @@ -24,7 +24,7 @@ services: # Only needed for bootstrap command: sh -c "chown -R node:node /home/node/shared && exec ./scripts/entrypoint.sh" healthcheck: - test: ["CMD", "node", "-e", "http.get('http://localhost:3000/healthcheck', res => process.exit(res.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] + test: ["CMD", "node", "-e", "http.get('http://localhost:8030/healthcheck', res => process.exit(res.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] interval: 30s timeout: 10s retries: 5 diff --git a/hosting/docker/worker/docker-compose.yml b/hosting/docker/worker/docker-compose.yml index 0d924916b39..99742bf61ae 100644 --- a/hosting/docker/worker/docker-compose.yml +++ b/hosting/docker/worker/docker-compose.yml @@ -14,10 +14,6 @@ services: logging: *logging-config depends_on: - docker-proxy - networks: - - supervisor - - docker-proxy - - webapp volumes: - shared:/home/node/shared # Only needed for bootstrap @@ -59,8 +55,6 @@ services: logging: *logging-config volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - networks: - - docker-proxy environment: - LOG_LEVEL=info - POST=1 @@ -76,12 +70,4 @@ services: start_period: 5s volumes: - shared: - -networks: - docker-proxy: - name: docker-proxy - supervisor: - name: supervisor - webapp: - name: webapp + shared: \ No newline at end of file From 226aa2a20128359e9cda6c76d8994d8003b0b598 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月13日 16:57:41 +0930 Subject: [PATCH 21/43] fix: ports --- hosting/docker/webapp/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index b0ec90b2ac0..5577d598101 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -13,7 +13,7 @@ services: restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config ports: - - "8030:3000" + - 3000 depends_on: - redis - clickhouse From 8e9fe093e17f731396b9bcc7518ffa838f7ab566 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月13日 17:31:36 +0930 Subject: [PATCH 22/43] fix: goose migration script --- docker/scripts/entrypoint.sh | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/docker/scripts/entrypoint.sh b/docker/scripts/entrypoint.sh index 102af5bd31e..f74d35b2307 100755 --- a/docker/scripts/entrypoint.sh +++ b/docker/scripts/entrypoint.sh @@ -15,16 +15,31 @@ if [ -n "$CLICKHOUSE_URL" ]; then echo "Running ClickHouse migrations..." export GOOSE_DRIVER=clickhouse - # Ensure secure=true is in the connection string - if echo "$CLICKHOUSE_URL" | grep -q "secure="; then - # secure parameter already exists, use as is - export GOOSE_DBSTRING="$CLICKHOUSE_URL" - elif echo "$CLICKHOUSE_URL" | grep -q "?"; then - # URL has query parameters, append secure=true - export GOOSE_DBSTRING="${CLICKHOUSE_URL}&secure=true" + # Extract host and credentials from CLICKHOUSE_URL (http://user:pass@host:8123) + # Convert to goose format (tcp://user:pass@host:9000) + if [ -n "$GOOSE_DBSTRING" ]; then + # If GOOSE_DBSTRING is explicitly set, use it + echo "Using provided GOOSE_DBSTRING" else - # URL has no query parameters, add secure=true - export GOOSE_DBSTRING="${CLICKHOUSE_URL}?secure=true" + # Extract components from CLICKHOUSE_URL and build tcp connection string + # Pattern: http://user:password@host:8123 -> tcp://user:password@host:9000 + CLICKHOUSE_HOST=$(echo "$CLICKHOUSE_URL" | sed -E 's|https?://([^:]+):([^@]+)@([^:]+):.*|3円|') + CLICKHOUSE_USER=$(echo "$CLICKHOUSE_URL" | sed -E 's|https?://([^:]+):([^@]+)@.*|1円|') + CLICKHOUSE_PASS=$(echo "$CLICKHOUSE_URL" | sed -E 's|https?://([^:]+):([^@]+)@.*|2円|') + + # Default to clickhouse:9000 if extraction fails + if [ -z "$CLICKHOUSE_HOST" ]; then + CLICKHOUSE_HOST="clickhouse" + fi + if [ -z "$CLICKHOUSE_USER" ]; then + CLICKHOUSE_USER="default" + fi + if [ -z "$CLICKHOUSE_PASS" ]; then + CLICKHOUSE_PASS="password" + fi + + export GOOSE_DBSTRING="tcp://${CLICKHOUSE_USER}:${CLICKHOUSE_PASS}@${CLICKHOUSE_HOST}:9000" + echo "Generated GOOSE_DBSTRING from CLICKHOUSE_URL" fi export GOOSE_MIGRATION_DIR=/triggerdotdev/internal-packages/clickhouse/schema From f4545f74ed7fd3a4ab481bad15e341057ba6f8da Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月13日 17:37:31 +0930 Subject: [PATCH 23/43] fix --- hosting/docker/webapp/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index 5577d598101..c3c18a2ca3c 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -59,6 +59,7 @@ services: TRIGGER_BOOTSTRAP_WORKER_TOKEN_PATH: /home/node/shared/worker_token # ClickHouse configuration CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://default:password@clickhouse:8123?secure=false} + GOOSE_DBSTRING: ${GOOSE_DBSTRING:-tcp://default:password@clickhouse:8123?secure=true} CLICKHOUSE_LOG_LEVEL: ${CLICKHOUSE_LOG_LEVEL:-info} # Run replication RUN_REPLICATION_ENABLED: ${RUN_REPLICATION_ENABLED:-1} From b48b3e057c119817c85e1c22fb05f9a51200d3d7 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月13日 17:45:42 +0930 Subject: [PATCH 24/43] fix: replace boot script --- hosting/docker/webapp/docker-compose.yml | 3 +- hosting/docker/webapp/scripts/entrypoint.sh | 66 +++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) create mode 100755 hosting/docker/webapp/scripts/entrypoint.sh diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index c3c18a2ca3c..bce034e5962 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -19,6 +19,7 @@ services: - clickhouse volumes: - shared:/home/node/shared + - ./scripts/entrypoint.sh:/triggerdotdev/scripts/entrypoint.sh:ro # Only needed for bootstrap user: root # Only needed for bootstrap @@ -59,7 +60,7 @@ services: TRIGGER_BOOTSTRAP_WORKER_TOKEN_PATH: /home/node/shared/worker_token # ClickHouse configuration CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://default:password@clickhouse:8123?secure=false} - GOOSE_DBSTRING: ${GOOSE_DBSTRING:-tcp://default:password@clickhouse:8123?secure=true} + GOOSE_DBSTRING: ${GOOSE_DBSTRING:-tcp://default:password@clickhouse:9000} CLICKHOUSE_LOG_LEVEL: ${CLICKHOUSE_LOG_LEVEL:-info} # Run replication RUN_REPLICATION_ENABLED: ${RUN_REPLICATION_ENABLED:-1} diff --git a/hosting/docker/webapp/scripts/entrypoint.sh b/hosting/docker/webapp/scripts/entrypoint.sh new file mode 100755 index 00000000000..f74d35b2307 --- /dev/null +++ b/hosting/docker/webapp/scripts/entrypoint.sh @@ -0,0 +1,66 @@ +#!/bin/sh +set -xe + +if [ -n "$DATABASE_HOST" ]; then + scripts/wait-for-it.sh ${DATABASE_HOST} -- echo "database is up" +fi + +# Run migrations +echo "Running prisma migrations" +pnpm --filter @trigger.dev/database db:migrate:deploy +echo "Prisma migrations done" + +if [ -n "$CLICKHOUSE_URL" ]; then + # Run ClickHouse migrations + echo "Running ClickHouse migrations..." + export GOOSE_DRIVER=clickhouse + + # Extract host and credentials from CLICKHOUSE_URL (http://user:pass@host:8123) + # Convert to goose format (tcp://user:pass@host:9000) + if [ -n "$GOOSE_DBSTRING" ]; then + # If GOOSE_DBSTRING is explicitly set, use it + echo "Using provided GOOSE_DBSTRING" + else + # Extract components from CLICKHOUSE_URL and build tcp connection string + # Pattern: http://user:password@host:8123 -> tcp://user:password@host:9000 + CLICKHOUSE_HOST=$(echo "$CLICKHOUSE_URL" | sed -E 's|https?://([^:]+):([^@]+)@([^:]+):.*|3円|') + CLICKHOUSE_USER=$(echo "$CLICKHOUSE_URL" | sed -E 's|https?://([^:]+):([^@]+)@.*|1円|') + CLICKHOUSE_PASS=$(echo "$CLICKHOUSE_URL" | sed -E 's|https?://([^:]+):([^@]+)@.*|2円|') + + # Default to clickhouse:9000 if extraction fails + if [ -z "$CLICKHOUSE_HOST" ]; then + CLICKHOUSE_HOST="clickhouse" + fi + if [ -z "$CLICKHOUSE_USER" ]; then + CLICKHOUSE_USER="default" + fi + if [ -z "$CLICKHOUSE_PASS" ]; then + CLICKHOUSE_PASS="password" + fi + + export GOOSE_DBSTRING="tcp://${CLICKHOUSE_USER}:${CLICKHOUSE_PASS}@${CLICKHOUSE_HOST}:9000" + echo "Generated GOOSE_DBSTRING from CLICKHOUSE_URL" + fi + + export GOOSE_MIGRATION_DIR=/triggerdotdev/internal-packages/clickhouse/schema + /usr/local/bin/goose up + echo "ClickHouse migrations complete." +else + echo "CLICKHOUSE_URL not set, skipping ClickHouse migrations." +fi + +# Copy over required prisma files +cp internal-packages/database/prisma/schema.prisma apps/webapp/prisma/ +cp node_modules/@prisma/engines/*.node apps/webapp/prisma/ + +cd /triggerdotdev/apps/webapp + + +# Decide how much old-space memory Node should get. +# Use $NODE_MAX_OLD_SPACE_SIZE if it’s set; otherwise fall back to 8192. +MAX_OLD_SPACE_SIZE="${NODE_MAX_OLD_SPACE_SIZE:-8192}" + +echo "Setting max old space size to ${MAX_OLD_SPACE_SIZE}" + +NODE_PATH='/triggerdotdev/node_modules/.pnpm/node_modules' exec dumb-init node --max-old-space-size=${MAX_OLD_SPACE_SIZE} ./build/server.js + From b3984ac3e0bed939b4669cb500285582cad6bb79 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月13日 18:01:16 +0930 Subject: [PATCH 25/43] fix: add network --- hosting/docker/webapp/docker-compose.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index bce034e5962..f05cafa015c 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -14,6 +14,8 @@ services: logging: *logging-config ports: - 3000 + networks: + - dokploy-network depends_on: - redis - clickhouse @@ -84,8 +86,8 @@ services: image: redis:${REDIS_IMAGE_TAG:-7} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config - ports: - - ${REDIS_PUBLISH_IP:-127.0.0.1}:6389:6379 + networks: + - dokploy-network volumes: - redis:/data healthcheck: @@ -99,6 +101,8 @@ services: image: electricsql/electric:${ELECTRIC_IMAGE_TAG:-1.0.24} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config + networks: + - dokploy-network environment: DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/main?schema=public&sslmode=disable} ELECTRIC_INSECURE: true @@ -114,6 +118,8 @@ services: image: bitnami/clickhouse:${CLICKHOUSE_IMAGE_TAG:-latest} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config + networks: + - dokploy-network environment: CLICKHOUSE_ADMIN_USER: ${CLICKHOUSE_USER:-default} CLICKHOUSE_ADMIN_PASSWORD: ${CLICKHOUSE_PASSWORD:-password} @@ -130,4 +136,8 @@ services: volumes: clickhouse: redis: - shared: \ No newline at end of file + shared: + +networks: + dokploy-network: + external: true \ No newline at end of file From b60099ad449b7866819829a4090cb77d1f170896 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月13日 18:14:19 +0930 Subject: [PATCH 26/43] fix: update network --- hosting/docker/webapp/docker-compose.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index f05cafa015c..d987905d085 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -13,9 +13,9 @@ services: restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config ports: - - 3000 + - 8030:3000 networks: - - dokploy-network + - webapp depends_on: - redis - clickhouse @@ -87,7 +87,7 @@ services: restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config networks: - - dokploy-network + - webapp volumes: - redis:/data healthcheck: @@ -102,7 +102,7 @@ services: restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config networks: - - dokploy-network + - webapp environment: DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/main?schema=public&sslmode=disable} ELECTRIC_INSECURE: true @@ -119,7 +119,7 @@ services: restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config networks: - - dokploy-network + - webapp environment: CLICKHOUSE_ADMIN_USER: ${CLICKHOUSE_USER:-default} CLICKHOUSE_ADMIN_PASSWORD: ${CLICKHOUSE_PASSWORD:-password} @@ -139,5 +139,4 @@ volumes: shared: networks: - dokploy-network: - external: true \ No newline at end of file + webapp: \ No newline at end of file From 46357356c914977bd172702e58a6f296ac4cc401 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月13日 18:26:47 +0930 Subject: [PATCH 27/43] fix: port --- hosting/docker/webapp/docker-compose.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index d987905d085..322192ddc5e 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -12,8 +12,6 @@ services: image: ghcr.io/triggerdotdev/trigger.dev:${TRIGGER_IMAGE_TAG:-v4-beta} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config - ports: - - 8030:3000 networks: - webapp depends_on: @@ -27,7 +25,7 @@ services: # Only needed for bootstrap command: sh -c "chown -R node:node /home/node/shared && exec ./scripts/entrypoint.sh" healthcheck: - test: ["CMD", "node", "-e", "http.get('http://localhost:8030/healthcheck', res => process.exit(res.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] + test: ["CMD", "node", "-e", "http.get('http://localhost:3000/healthcheck', res => process.exit(res.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] interval: 30s timeout: 10s retries: 5 From 94d95a85fbebc03ef1ed136fcebf2efe813b8ab6 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月13日 22:05:03 +0930 Subject: [PATCH 28/43] fix: coolify --- hosting/docker/webapp/docker-compose.yml | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index 322192ddc5e..cdfca409078 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -12,18 +12,21 @@ services: image: ghcr.io/triggerdotdev/trigger.dev:${TRIGGER_IMAGE_TAG:-v4-beta} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config - networks: - - webapp depends_on: - redis - clickhouse + expose: + - "3000" volumes: - shared:/home/node/shared - - ./scripts/entrypoint.sh:/triggerdotdev/scripts/entrypoint.sh:ro + - type: bind + source: ./scripts/entrypoint.sh + target: /entrypoint.sh # ✅ 单文件挂载到根 + read_only: true # Only needed for bootstrap user: root # Only needed for bootstrap - command: sh -c "chown -R node:node /home/node/shared && exec ./scripts/entrypoint.sh" + command: sh -c "chown -R node:node /home/node/shared && exec entrypoint.sh" healthcheck: test: ["CMD", "node", "-e", "http.get('http://localhost:3000/healthcheck', res => process.exit(res.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] interval: 30s @@ -84,8 +87,6 @@ services: image: redis:${REDIS_IMAGE_TAG:-7} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config - networks: - - webapp volumes: - redis:/data healthcheck: @@ -99,8 +100,6 @@ services: image: electricsql/electric:${ELECTRIC_IMAGE_TAG:-1.0.24} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config - networks: - - webapp environment: DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/main?schema=public&sslmode=disable} ELECTRIC_INSECURE: true @@ -116,8 +115,6 @@ services: image: bitnami/clickhouse:${CLICKHOUSE_IMAGE_TAG:-latest} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config - networks: - - webapp environment: CLICKHOUSE_ADMIN_USER: ${CLICKHOUSE_USER:-default} CLICKHOUSE_ADMIN_PASSWORD: ${CLICKHOUSE_PASSWORD:-password} @@ -134,7 +131,4 @@ services: volumes: clickhouse: redis: - shared: - -networks: - webapp: \ No newline at end of file + shared: \ No newline at end of file From cbc7e01b16f8e3c808c69191ec3e4d7c986ab590 Mon Sep 17 00:00:00 2001 From: dqaria Date: 2025年8月13日 22:48:45 +0930 Subject: [PATCH 29/43] Update docker-compose.yml --- hosting/docker/webapp/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index cdfca409078..965ac17369c 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -26,7 +26,7 @@ services: # Only needed for bootstrap user: root # Only needed for bootstrap - command: sh -c "chown -R node:node /home/node/shared && exec entrypoint.sh" + command: sh -c "chown -R node:node /home/node/shared && exec /entrypoint.sh" healthcheck: test: ["CMD", "node", "-e", "http.get('http://localhost:3000/healthcheck', res => process.exit(res.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] interval: 30s From 01789d8b040051a30a66d7cc4d26ce0c4bd6a268 Mon Sep 17 00:00:00 2001 From: dqaria Date: 2025年8月13日 22:51:24 +0930 Subject: [PATCH 30/43] Update docker-compose.yml --- hosting/docker/webapp/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index 965ac17369c..d0c095483c8 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -26,7 +26,7 @@ services: # Only needed for bootstrap user: root # Only needed for bootstrap - command: sh -c "chown -R node:node /home/node/shared && exec /entrypoint.sh" + command: sh -c "chown -R node:node /home/node/shared && exec sh /entrypoint.sh" healthcheck: test: ["CMD", "node", "-e", "http.get('http://localhost:3000/healthcheck', res => process.exit(res.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] interval: 30s From c1dca18405df70fc92ae90f1dcdd6600e1ac797e Mon Sep 17 00:00:00 2001 From: dqaria Date: 2025年8月13日 23:00:46 +0930 Subject: [PATCH 31/43] Update docker-compose.yml --- hosting/docker/webapp/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index d0c095483c8..f8658c26a1e 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -26,7 +26,7 @@ services: # Only needed for bootstrap user: root # Only needed for bootstrap - command: sh -c "chown -R node:node /home/node/shared && exec sh /entrypoint.sh" + command: sh -c "chown -R node:node /home/node/shared && cp /entrypoint.sh /tmp/entrypoint.sh && chmod +x /tmp/entrypoint.sh && exec /tmp/entrypoint.sh" healthcheck: test: ["CMD", "node", "-e", "http.get('http://localhost:3000/healthcheck', res => process.exit(res.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] interval: 30s From 9c93d88baffc8d2436cc1a382b6571d4e8b3821f Mon Sep 17 00:00:00 2001 From: dqaria Date: 2025年8月13日 23:09:57 +0930 Subject: [PATCH 32/43] Update docker-compose.yml --- hosting/docker/webapp/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index f8658c26a1e..f9ceea187b2 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -21,12 +21,12 @@ services: - shared:/home/node/shared - type: bind source: ./scripts/entrypoint.sh - target: /entrypoint.sh # ✅ 单文件挂载到根 + target: /triggerdotdev/entrypoint.sh read_only: true # Only needed for bootstrap user: root # Only needed for bootstrap - command: sh -c "chown -R node:node /home/node/shared && cp /entrypoint.sh /tmp/entrypoint.sh && chmod +x /tmp/entrypoint.sh && exec /tmp/entrypoint.sh" + command: sh -c "chown -R node:node /home/node/shared && exec /triggerdotdev/entrypoint.sh" healthcheck: test: ["CMD", "node", "-e", "http.get('http://localhost:3000/healthcheck', res => process.exit(res.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] interval: 30s From 1b5edfe58b654167ba174bb293e6d282fb95f24c Mon Sep 17 00:00:00 2001 From: dqaria Date: 2025年8月13日 23:14:20 +0930 Subject: [PATCH 33/43] Update docker-compose.yml --- hosting/docker/webapp/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index f9ceea187b2..7290d7d983f 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -26,7 +26,7 @@ services: # Only needed for bootstrap user: root # Only needed for bootstrap - command: sh -c "chown -R node:node /home/node/shared && exec /triggerdotdev/entrypoint.sh" + command: sh -c "chown -R node:node /home/node/shared" healthcheck: test: ["CMD", "node", "-e", "http.get('http://localhost:3000/healthcheck', res => process.exit(res.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] interval: 30s From daf9d1197cdad065b0be7a25a4eb05e64dbc4eb1 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月14日 10:12:48 +0930 Subject: [PATCH 34/43] fix: add back --- hosting/docker/webapp/docker-compose.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index 7290d7d983f..b488e530d2b 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -15,18 +15,15 @@ services: depends_on: - redis - clickhouse - expose: - - "3000" + networks: + - webapp volumes: - shared:/home/node/shared - - type: bind - source: ./scripts/entrypoint.sh - target: /triggerdotdev/entrypoint.sh - read_only: true + - ./scripts/entrypoint.sh:/triggerdotdev/scripts/entrypoint.sh:ro # Only needed for bootstrap user: root # Only needed for bootstrap - command: sh -c "chown -R node:node /home/node/shared" + command: sh -c "chown -R node:node /home/node/shared && exec ./scripts/entrypoint.sh" healthcheck: test: ["CMD", "node", "-e", "http.get('http://localhost:3000/healthcheck', res => process.exit(res.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] interval: 30s @@ -87,6 +84,8 @@ services: image: redis:${REDIS_IMAGE_TAG:-7} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config + networks: + - webapp volumes: - redis:/data healthcheck: @@ -100,6 +99,8 @@ services: image: electricsql/electric:${ELECTRIC_IMAGE_TAG:-1.0.24} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config + networks: + - webapp environment: DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/main?schema=public&sslmode=disable} ELECTRIC_INSECURE: true @@ -115,6 +116,8 @@ services: image: bitnami/clickhouse:${CLICKHOUSE_IMAGE_TAG:-latest} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config + networks: + - webapp environment: CLICKHOUSE_ADMIN_USER: ${CLICKHOUSE_USER:-default} CLICKHOUSE_ADMIN_PASSWORD: ${CLICKHOUSE_PASSWORD:-password} @@ -131,4 +134,7 @@ services: volumes: clickhouse: redis: - shared: \ No newline at end of file + shared: + +networks: + webapp: \ No newline at end of file From e2758f6dee87a8248b50b32718e8e657a0e6df74 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月14日 10:55:37 +0930 Subject: [PATCH 35/43] fix: tls --- hosting/docker/webapp/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index b488e530d2b..29bb15bfb68 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -60,7 +60,7 @@ services: TRIGGER_BOOTSTRAP_WORKER_TOKEN_PATH: /home/node/shared/worker_token # ClickHouse configuration CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://default:password@clickhouse:8123?secure=false} - GOOSE_DBSTRING: ${GOOSE_DBSTRING:-tcp://default:password@clickhouse:9000} + GOOSE_DBSTRING: ${GOOSE_DBSTRING:-tcp://default:password@clickhouse:9000?secure=true} CLICKHOUSE_LOG_LEVEL: ${CLICKHOUSE_LOG_LEVEL:-info} # Run replication RUN_REPLICATION_ENABLED: ${RUN_REPLICATION_ENABLED:-1} From 0a76aa6c0162f7a058e4fb51d68c2c6be0c300b8 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月14日 10:57:58 +0930 Subject: [PATCH 36/43] fix --- hosting/docker/webapp/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index 29bb15bfb68..52c7f2c6910 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -60,7 +60,7 @@ services: TRIGGER_BOOTSTRAP_WORKER_TOKEN_PATH: /home/node/shared/worker_token # ClickHouse configuration CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://default:password@clickhouse:8123?secure=false} - GOOSE_DBSTRING: ${GOOSE_DBSTRING:-tcp://default:password@clickhouse:9000?secure=true} + GOOSE_DBSTRING: ${GOOSE_DBSTRING:-tcp://default:password@clickhouse:9000?secure=false} CLICKHOUSE_LOG_LEVEL: ${CLICKHOUSE_LOG_LEVEL:-info} # Run replication RUN_REPLICATION_ENABLED: ${RUN_REPLICATION_ENABLED:-1} From 9405aa3e1a24179cc81d5fe6ca35324d19c954b3 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月14日 11:00:23 +0930 Subject: [PATCH 37/43] fix --- hosting/docker/webapp/docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index 52c7f2c6910..4c0a27fade5 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -60,7 +60,6 @@ services: TRIGGER_BOOTSTRAP_WORKER_TOKEN_PATH: /home/node/shared/worker_token # ClickHouse configuration CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://default:password@clickhouse:8123?secure=false} - GOOSE_DBSTRING: ${GOOSE_DBSTRING:-tcp://default:password@clickhouse:9000?secure=false} CLICKHOUSE_LOG_LEVEL: ${CLICKHOUSE_LOG_LEVEL:-info} # Run replication RUN_REPLICATION_ENABLED: ${RUN_REPLICATION_ENABLED:-1} From 6313c9a698d39b68c723310c9641e2c2fea74c62 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月14日 16:46:47 +0930 Subject: [PATCH 38/43] fix: network --- hosting/docker/worker/docker-compose.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/hosting/docker/worker/docker-compose.yml b/hosting/docker/worker/docker-compose.yml index 99742bf61ae..82fdb9691d6 100644 --- a/hosting/docker/worker/docker-compose.yml +++ b/hosting/docker/worker/docker-compose.yml @@ -14,6 +14,9 @@ services: logging: *logging-config depends_on: - docker-proxy + networks: + - supervisor + - docker-proxy volumes: - shared:/home/node/shared # Only needed for bootstrap @@ -55,6 +58,8 @@ services: logging: *logging-config volumes: - /var/run/docker.sock:/var/run/docker.sock:ro + networks: + - docker-proxy environment: - LOG_LEVEL=info - POST=1 @@ -70,4 +75,10 @@ services: start_period: 5s volumes: - shared: \ No newline at end of file + shared: + +networks: + docker-proxy: + name: docker-proxy + supervisor: + name: supervisor \ No newline at end of file From 7b7511beb50b9d665302e85febdc45974ff3a0b6 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月14日 17:23:34 +0930 Subject: [PATCH 39/43] fix --- hosting/docker/worker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/docker/worker/docker-compose.yml b/hosting/docker/worker/docker-compose.yml index 82fdb9691d6..94e1bcac7f4 100644 --- a/hosting/docker/worker/docker-compose.yml +++ b/hosting/docker/worker/docker-compose.yml @@ -9,7 +9,7 @@ x-logging: &logging-config services: supervisor: - image: ghcr.io/lattebit/supervisor:latest + image: ghcr.io/triggerdotdev/supervisor:${TRIGGER_IMAGE_TAG:-v4-beta} restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config depends_on: From f3a2b18cd71a12e6cd904cf47b4f404d5b244080 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月14日 18:49:50 +0930 Subject: [PATCH 40/43] fix --- hosting/docker/worker/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hosting/docker/worker/docker-compose.yml b/hosting/docker/worker/docker-compose.yml index 94e1bcac7f4..599abe40c8b 100644 --- a/hosting/docker/worker/docker-compose.yml +++ b/hosting/docker/worker/docker-compose.yml @@ -9,7 +9,7 @@ x-logging: &logging-config services: supervisor: - image: ghcr.io/triggerdotdev/supervisor:${TRIGGER_IMAGE_TAG:-v4-beta} + image: ghcr.io/lattebit/supervisor:latest restart: ${RESTART_POLICY:-unless-stopped} logging: *logging-config depends_on: @@ -33,7 +33,7 @@ services: OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://webapp:3000/otel} TRIGGER_WORKLOAD_API_DOMAIN: supervisor TRIGGER_WORKLOAD_API_PORT_EXTERNAL: 8020 - #RESOURCE_MONITOR_ENABLED: 1 + RESOURCE_MONITOR_ENABLED: ${TRIGGER_WORKER_MAX_RUN_COUNT:-1} TRIGGER_WORKER_MAX_RUN_COUNT: ${TRIGGER_WORKER_MAX_RUN_COUNT:-2} # Optional settings DEBUG: 1 From 2b76aaa89fd6725f80a5cacd14ca281144533405 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月14日 19:09:51 +0930 Subject: [PATCH 41/43] feat: roolback --- apps/supervisor/src/index.ts | 41 +++--------------------- hosting/docker/worker/docker-compose.yml | 2 +- 2 files changed, 5 insertions(+), 38 deletions(-) diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts index 2f4aab60db7..afccb74605d 100644 --- a/apps/supervisor/src/index.ts +++ b/apps/supervisor/src/index.ts @@ -31,7 +31,6 @@ if (env.METRICS_COLLECT_DEFAULTS) { } class ManagedSupervisor { - private activeTaskCount = 0; private readonly workerSession: SupervisorSession; private readonly metricsServer?: HttpServer; private readonly workloadServer: WorkloadServer; @@ -134,15 +133,6 @@ class ManagedSupervisor { heartbeatIntervalSeconds: env.TRIGGER_WORKER_HEARTBEAT_INTERVAL_SECONDS, sendRunDebugLogs: env.SEND_RUN_DEBUG_LOGS, preDequeue: async () => { - // 任务数量限制检查(优先级最高) - if (this.activeTaskCount>= env.TRIGGER_WORKER_MAX_RUN_COUNT) { - this.logger.debug("Skipping dequeue due to activeTaskCount limit", { - activeTaskCount: this.activeTaskCount, - maxRunCount: env.TRIGGER_WORKER_MAX_RUN_COUNT, - }); - return { skipDequeue: true }; - } - if (!env.RESOURCE_MONITOR_ENABLED) { return {}; } @@ -154,20 +144,12 @@ class ManagedSupervisor { const resources = await this.resourceMonitor.getNodeResources(); - const shouldSkip = resources.cpuAvailable < 0.25 || resources.memoryAvailable < 0.25; - if (shouldSkip) { - this.logger.debug("Skipping dequeue due to resource constraints", { - cpuAvailable: resources.cpuAvailable, - memoryAvailable: resources.memoryAvailable, - }); - } - return { maxResources: { cpu: resources.cpuAvailable, memory: resources.memoryAvailable, }, - skipDequeue: shouldSkip, + skipDequeue: resources.cpuAvailable < 0.25 || resources.memoryAvailable < 0.25, }; }, preSkip: async () => { @@ -196,7 +178,6 @@ class ManagedSupervisor { this.workerSession.on("runQueueMessage", async ({ time, message }) => { this.logger.log(`Received message with timestamp ${time.toLocaleString()}`, message); - this.logger.log(`Active tasks before processing: ${this.activeTaskCount}/${env.TRIGGER_WORKER_MAX_RUN_COUNT}`); if (message.completedWaitpoints.length> 0) { this.logger.debug("Run has completed waitpoints", { @@ -248,8 +229,6 @@ class ManagedSupervisor { if (didWarmStart) { this.logger.log("Warm start successful", { runId: message.run.id }); - this.activeTaskCount++; - this.logger.log(`Active tasks after warm start: ${this.activeTaskCount}/${env.TRIGGER_WORKER_MAX_RUN_COUNT}`); return; } @@ -270,9 +249,6 @@ class ManagedSupervisor { snapshotFriendlyId: message.snapshot.friendlyId, }); - this.activeTaskCount++; - this.logger.log(`Active tasks after creating workload: ${this.activeTaskCount}/${env.TRIGGER_WORKER_MAX_RUN_COUNT}`); - // Disabled for now // this.resourceMonitor.blockResources({ // cpu: message.run.machine.cpu, @@ -307,21 +283,12 @@ class ManagedSupervisor { } async onRunConnected({ run }: { run: { friendlyId: string } }) { - this.logger.debug("Run connected", { - run, - activeTaskCount: this.activeTaskCount, - maxRunCount: env.TRIGGER_WORKER_MAX_RUN_COUNT - }); + this.logger.debug("Run connected", { run }); this.workerSession.subscribeToRunNotifications([run.friendlyId]); } async onRunDisconnected({ run }: { run: { friendlyId: string } }) { - this.activeTaskCount = Math.max(0, this.activeTaskCount - 1); - this.logger.debug("Run disconnected", { - run, - activeTaskCount: this.activeTaskCount, - maxRunCount: env.TRIGGER_WORKER_MAX_RUN_COUNT - }); + this.logger.debug("Run disconnected", { run }); this.workerSession.unsubscribeFromRunNotifications([run.friendlyId]); } @@ -403,4 +370,4 @@ class ManagedSupervisor { } const worker = new ManagedSupervisor(); -worker.start(); +worker.start(); \ No newline at end of file diff --git a/hosting/docker/worker/docker-compose.yml b/hosting/docker/worker/docker-compose.yml index 599abe40c8b..5a01da81f61 100644 --- a/hosting/docker/worker/docker-compose.yml +++ b/hosting/docker/worker/docker-compose.yml @@ -40,7 +40,7 @@ services: ENFORCE_MACHINE_PRESETS: 1 TRIGGER_DEQUEUE_INTERVAL_MS: 1000 DOCKER_HOST: tcp://docker-proxy:2375 - DOCKER_RUNNER_NETWORKS: webapp,supervisor + DOCKER_RUNNER_NETWORKS: supervisor DOCKER_REGISTRY_URL: ${DOCKER_REGISTRY_URL:-localhost:5000} # DOCKER_REGISTRY_USERNAME: ${DOCKER_REGISTRY_USERNAME:-} # DOCKER_REGISTRY_PASSWORD: ${DOCKER_REGISTRY_PASSWORD:-} From 555e03922a4ebbab9238a9685a47e0165f7c8a6b Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月14日 19:32:08 +0930 Subject: [PATCH 42/43] feat: add new activeRuns logic --- apps/supervisor/src/index.ts | 36 +++++++++++++++++++++ apps/supervisor/src/workloadServer/index.ts | 8 +++++ scripts/build-worker.sh | 14 ++++++++ 3 files changed, 58 insertions(+) create mode 100755 scripts/build-worker.sh diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts index afccb74605d..e5fb194ddf4 100644 --- a/apps/supervisor/src/index.ts +++ b/apps/supervisor/src/index.ts @@ -44,6 +44,7 @@ class ManagedSupervisor { private readonly isKubernetes = isKubernetesEnvironment(env.KUBERNETES_FORCE_ENABLED); private readonly warmStartUrl = env.TRIGGER_WARM_START_URL; + private readonly maxRunCount = env.TRIGGER_WORKER_MAX_RUN_COUNT; constructor() { const { TRIGGER_WORKER_TOKEN, MANAGED_WORKER_SECRET, ...envWithoutSecrets } = env; @@ -133,6 +134,27 @@ class ManagedSupervisor { heartbeatIntervalSeconds: env.TRIGGER_WORKER_HEARTBEAT_INTERVAL_SECONDS, sendRunDebugLogs: env.SEND_RUN_DEBUG_LOGS, preDequeue: async () => { + // Check if we've reached the maximum number of running tasks by querying WorkloadServer + const currentRunCount = this.workloadServer.getActiveRunCount(); + const activeRuns = this.workloadServer.getActiveRuns(); + + this.logger.debug("Checking task count before dequeue", { + currentRunCount, + maxRunCount: this.maxRunCount, + activeRuns, + }); + + // Skip dequeue if we've reached the maximum run count + if (currentRunCount>= this.maxRunCount) { + this.logger.log("Skipping dequeue: max run count reached", { + currentRunCount, + maxRunCount: this.maxRunCount, + }); + return { + skipDequeue: true, + }; + } + if (!env.RESOURCE_MONITOR_ENABLED) { return {}; } @@ -285,11 +307,25 @@ class ManagedSupervisor { async onRunConnected({ run }: { run: { friendlyId: string } }) { this.logger.debug("Run connected", { run }); this.workerSession.subscribeToRunNotifications([run.friendlyId]); + + const currentRunCount = this.workloadServer.getActiveRunCount(); + this.logger.log("Task started", { + runId: run.friendlyId, + currentRunCount, + maxRunCount: this.maxRunCount, + }); } async onRunDisconnected({ run }: { run: { friendlyId: string } }) { this.logger.debug("Run disconnected", { run }); this.workerSession.unsubscribeFromRunNotifications([run.friendlyId]); + + const currentRunCount = this.workloadServer.getActiveRunCount(); + this.logger.log("Task completed", { + runId: run.friendlyId, + currentRunCount, + maxRunCount: this.maxRunCount, + }); } private async tryWarmStart(dequeuedMessage: DequeuedMessage): Promise { diff --git a/apps/supervisor/src/workloadServer/index.ts b/apps/supervisor/src/workloadServer/index.ts index e7e391bce38..282218e337e 100644 --- a/apps/supervisor/src/workloadServer/index.ts +++ b/apps/supervisor/src/workloadServer/index.ts @@ -571,6 +571,14 @@ export class WorkloadServer extends EventEmitter { return websocketServer; } + getActiveRunCount(): number { + return this.runSockets.size; + } + + getActiveRuns(): string[] { + return Array.from(this.runSockets.keys()); + } + notifyRun({ run }: { run: { friendlyId: string } }) { try { const runSocket = this.runSockets.get(run.friendlyId); diff --git a/scripts/build-worker.sh b/scripts/build-worker.sh new file mode 100755 index 00000000000..2227c4c4a32 --- /dev/null +++ b/scripts/build-worker.sh @@ -0,0 +1,14 @@ +IMAGE=ghcr.io/lattebit/supervisor + +# 自动生成不可变标签:2025年08月14日_1015-abc123 +DATE=$(date -u +%Y%m%d_%H%M) +SHA=$(git rev-parse --short HEAD) +AUTO_TAG="${DATE}-${SHA}" + +# 构建并同时推 latest 和 自动标签 +docker buildx build \ + --platform linux/amd64,linux/arm64 \ + -f apps/supervisor/Containerfile \ + -t $IMAGE:latest \ + -t $IMAGE:${AUTO_TAG} \ + --push . \ No newline at end of file From 0a3c1da4ab2987dc454236d0e5207fc866feba94 Mon Sep 17 00:00:00 2001 From: Daqi Song Date: 2025年8月14日 20:06:54 +0930 Subject: [PATCH 43/43] feat: add lock --- apps/supervisor/src/index.ts | 19 +++++++ apps/supervisor/src/workloadServer/index.ts | 57 ++++++++++++++++++++- 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts index e5fb194ddf4..b809a240fdf 100644 --- a/apps/supervisor/src/index.ts +++ b/apps/supervisor/src/index.ts @@ -142,6 +142,7 @@ class ManagedSupervisor { currentRunCount, maxRunCount: this.maxRunCount, activeRuns, + note: "currentRunCount includes both connected and pending runs", }); // Skip dequeue if we've reached the maximum run count @@ -149,6 +150,7 @@ class ManagedSupervisor { this.logger.log("Skipping dequeue: max run count reached", { currentRunCount, maxRunCount: this.maxRunCount, + note: "Includes pending runs to prevent over-dequeuing", }); return { skipDequeue: true, @@ -201,6 +203,9 @@ class ManagedSupervisor { this.workerSession.on("runQueueMessage", async ({ time, message }) => { this.logger.log(`Received message with timestamp ${time.toLocaleString()}`, message); + // Immediately reserve the run slot to prevent over-dequeuing + this.workloadServer.reservePendingRun(message.run.friendlyId); + if (message.completedWaitpoints.length> 0) { this.logger.debug("Run has completed waitpoints", { runId: message.run.id, @@ -210,6 +215,8 @@ class ManagedSupervisor { if (!message.image) { this.logger.error("Run has no image", { runId: message.run.id }); + // Release the reservation if we can't process the run + this.workloadServer.releasePendingRun(message.run.friendlyId); return; } @@ -220,6 +227,8 @@ class ManagedSupervisor { if (!this.checkpointClient) { this.logger.error("No checkpoint client", { runId: message.run.id }); + // Release the reservation since we can't restore + this.workloadServer.releasePendingRun(message.run.friendlyId); return; } @@ -235,6 +244,8 @@ class ManagedSupervisor { if (didRestore) { this.logger.log("Restore successful", { runId: message.run.id }); + // The restore process will handle the connection, so we can release here + // as the run will re-connect and be counted properly } else { this.logger.error("Restore failed", { runId: message.run.id }); } @@ -242,6 +253,8 @@ class ManagedSupervisor { this.logger.error("Failed to restore run", { error }); } + // Release the reservation after restore attempt + this.workloadServer.releasePendingRun(message.run.friendlyId); return; } @@ -251,6 +264,7 @@ class ManagedSupervisor { if (didWarmStart) { this.logger.log("Warm start successful", { runId: message.run.id }); + // Warm start handles the connection, pending reservation will be cleared when connected return; } @@ -271,6 +285,9 @@ class ManagedSupervisor { snapshotFriendlyId: message.snapshot.friendlyId, }); + // The workload will connect and the pending reservation will be cleared then + // If the workload fails to start, we should have a timeout to clean up pending reservations + // Disabled for now // this.resourceMonitor.blockResources({ // cpu: message.run.machine.cpu, @@ -278,6 +295,8 @@ class ManagedSupervisor { // }); } catch (error) { this.logger.error("Failed to create workload", { error }); + // Release the reservation if workload creation fails + this.workloadServer.releasePendingRun(message.run.friendlyId); } }); diff --git a/apps/supervisor/src/workloadServer/index.ts b/apps/supervisor/src/workloadServer/index.ts index 282218e337e..c385b1fed1b 100644 --- a/apps/supervisor/src/workloadServer/index.ts +++ b/apps/supervisor/src/workloadServer/index.ts @@ -84,6 +84,11 @@ export class WorkloadServer extends EventEmitter { > >(); + // Track pending runs that have been dequeued but not yet connected + private readonly pendingRuns = new Set(); + // Track timeouts for pending runs to prevent memory leaks + private readonly pendingRunTimeouts = new Map(); + private readonly workerClient: SupervisorHttpClient; constructor(opts: WorkloadServerOptions) { @@ -493,6 +498,14 @@ export class WorkloadServer extends EventEmitter { } this.runSockets.set(friendlyId, socket); + // Remove from pending when actually connected + this.pendingRuns.delete(friendlyId); + // Clear the timeout since the run has connected + const timeout = this.pendingRunTimeouts.get(friendlyId); + if (timeout) { + clearTimeout(timeout); + this.pendingRunTimeouts.delete(friendlyId); + } this.emit("runConnected", { run: { friendlyId } }); socket.data.runFriendlyId = friendlyId; }; @@ -501,6 +514,8 @@ export class WorkloadServer extends EventEmitter { socketLogger.debug("runDisconnected", { ...getSocketMetadata() }); this.runSockets.delete(friendlyId); + // Also remove from pending if it was there + this.pendingRuns.delete(friendlyId); this.emit("runDisconnected", { run: { friendlyId } }); socket.data.runFriendlyId = undefined; }; @@ -572,13 +587,53 @@ export class WorkloadServer extends EventEmitter { } getActiveRunCount(): number { - return this.runSockets.size; + // Count both connected runs and pending runs + return this.runSockets.size + this.pendingRuns.size; } getActiveRuns(): string[] { return Array.from(this.runSockets.keys()); } + reservePendingRun(runFriendlyId: string): void { + this.pendingRuns.add(runFriendlyId); + + // Set a timeout to clean up if the run doesn't connect within 5 minutes + const timeout = setTimeout(() => { + if (this.pendingRuns.has(runFriendlyId)) { + this.logger.warn("Pending run timed out, releasing reservation", { runFriendlyId }); + this.releasePendingRun(runFriendlyId); + } + }, 5 * 60 * 1000); // 5 minutes + + this.pendingRunTimeouts.set(runFriendlyId, timeout); + + this.logger.debug("Reserved pending run", { + runFriendlyId, + pendingCount: this.pendingRuns.size, + connectedCount: this.runSockets.size, + totalCount: this.getActiveRunCount() + }); + } + + releasePendingRun(runFriendlyId: string): void { + this.pendingRuns.delete(runFriendlyId); + + // Clear the timeout if it exists + const timeout = this.pendingRunTimeouts.get(runFriendlyId); + if (timeout) { + clearTimeout(timeout); + this.pendingRunTimeouts.delete(runFriendlyId); + } + + this.logger.debug("Released pending run", { + runFriendlyId, + pendingCount: this.pendingRuns.size, + connectedCount: this.runSockets.size, + totalCount: this.getActiveRunCount() + }); + } + notifyRun({ run }: { run: { friendlyId: string } }) { try { const runSocket = this.runSockets.get(run.friendlyId);

AltStyle によって変換されたページ (->オリジナル) /