From 3dc3057bf0d71e293a6b46485269d9f3df0392e8 Mon Sep 17 00:00:00 2001 From: Seungpyo1007 Date: Tue, 8 Sep 2026 20:39:38 +0900 Subject: [PATCH] fix(ci): give the dump its own schedule instead of the scrape's leftovers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #59 got the scrape to finish, and the run then died in the dump instead: Enrich 06:13 -> 10:33 260 min (on a 200-minute budget) Dump 10:39 -> 11:36 57 min, cut by the job timeout total 330 min Two things were wrong. The budget is only consulted *between* sources, so one long source overran it by an hour; each source is now capped at 40 minutes by `timeout`, and the budget drops to 150 so the rest of the pipeline keeps real room. But the dump does not belong behind a multi-hour live scrape at all. Whenever the scrape runs long the dump is what gets dropped, which is how the published pages drifted away from the records for six weeks. dump-refresh.yml already regenerates the dump from current data with no scraping — and when it last ran, the dump step succeeded; only the PR creation failed on a transient server error. Give it a weekly schedule (Tuesdays, a day after the Monday jobs) so the published dump is reconciled with data/ every week regardless of how the scrape goes, and a job timeout so it can never be silently truncated either. Refs #1 --- .github/workflows/dump-refresh.yml | 11 +++++++++++ .github/workflows/weekly-refresh.yml | 8 ++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dump-refresh.yml b/.github/workflows/dump-refresh.yml index c55c15f..edc2a9d 100644 --- a/.github/workflows/dump-refresh.yml +++ b/.github/workflows/dump-refresh.yml @@ -5,6 +5,14 @@ name: dump-refresh # after an engine-side change that affects serialized output (e.g. a scoring # version bump) — weekly-refresh.yml also does this but behind a slow live scrape. on: + # The dump has to be reconciled with data/ on a schedule of its own. + # weekly-refresh also regenerates it, but only after a multi-hour live + # scrape, and when that scrape runs long the dump is the step that gets + # dropped — which is how the published pages drifted away from the + # records for six weeks. Run a day after the Monday jobs so this picks + # up whatever they merged. + schedule: + - cron: "0 3 * * 2" # Tuesdays 03:00 UTC workflow_dispatch: permissions: @@ -17,6 +25,9 @@ concurrency: jobs: dump: runs-on: ubuntu-latest + # Well under GitHub's 6h ceiling, which reports an over-run as + # "cancelled" rather than failed and so hides the breakage. + timeout-minutes: 330 env: TECHAPI_WRITE_TOKEN: ${{ secrets.TECHENGINEBOT_TOKEN || secrets.TECHAPI_TOKEN }} # seed/validate/dump read the data tree from here. diff --git a/.github/workflows/weekly-refresh.yml b/.github/workflows/weekly-refresh.yml index c915f26..0dde21a 100644 --- a/.github/workflows/weekly-refresh.yml +++ b/.github/workflows/weekly-refresh.yml @@ -73,7 +73,10 @@ jobs: run_enrich() { comp="1ドル"; src="2ドル" echo "::group::enrich ${comp}/${src}" - if python -m app.ingest.enrich \ + # Cap each source: the budget below is only consulted between + # sources, so without this one slow source overruns it — the last + # run spent 260 minutes against a 200-minute budget that way. + if timeout "${PER_SOURCE_S}s" python -m app.ingest.enrich \ --source "$src" --component "$comp" \ --data-root ./techapi/data --sleep "$SLEEP" \ --summary "enrich-${comp}-${src}.md"; then @@ -88,7 +91,8 @@ jobs: # broken while validate, the static dump and the refresh PR were being # skipped every week. Give collection a budget it cannot exceed and # leave the remainder to the rest of the pipeline. - BUDGET_S=$(( 200 * 60 )) + BUDGET_S=$(( 150 * 60 )) + PER_SOURCE_S=$(( 40 * 60 )) DEADLINE=$(( $(date +%s) + BUDGET_S )) budget_left() { if [ "$(date +%s)" -ge "$DEADLINE" ]; then

AltStyle によって変換されたページ (->オリジナル) /