diff --git a/.github/workflows/dump-refresh.yml b/.github/workflows/dump-refresh.yml index c55c15f..edc2a9d 100644 --- a/.github/workflows/dump-refresh.yml +++ b/.github/workflows/dump-refresh.yml @@ -5,6 +5,14 @@ name: dump-refresh # after an engine-side change that affects serialized output (e.g. a scoring # version bump) — weekly-refresh.yml also does this but behind a slow live scrape. on: + # The dump has to be reconciled with data/ on a schedule of its own. + # weekly-refresh also regenerates it, but only after a multi-hour live + # scrape, and when that scrape runs long the dump is the step that gets + # dropped — which is how the published pages drifted away from the + # records for six weeks. Run a day after the Monday jobs so this picks + # up whatever they merged. + schedule: + - cron: "0 3 * * 2" # Tuesdays 03:00 UTC workflow_dispatch: permissions: @@ -17,6 +25,9 @@ concurrency: jobs: dump: runs-on: ubuntu-latest + # Well under GitHub's 6h ceiling, which reports an over-run as + # "cancelled" rather than failed and so hides the breakage. + timeout-minutes: 330 env: TECHAPI_WRITE_TOKEN: ${{ secrets.TECHENGINEBOT_TOKEN || secrets.TECHAPI_TOKEN }} # seed/validate/dump read the data tree from here. diff --git a/.github/workflows/weekly-refresh.yml b/.github/workflows/weekly-refresh.yml index c915f26..0dde21a 100644 --- a/.github/workflows/weekly-refresh.yml +++ b/.github/workflows/weekly-refresh.yml @@ -73,7 +73,10 @@ jobs: run_enrich() { comp="1ドル"; src="2ドル" echo "::group::enrich ${comp}/${src}" - if python -m app.ingest.enrich \ + # Cap each source: the budget below is only consulted between + # sources, so without this one slow source overruns it — the last + # run spent 260 minutes against a 200-minute budget that way. + if timeout "${PER_SOURCE_S}s" python -m app.ingest.enrich \ --source "$src" --component "$comp" \ --data-root ./techapi/data --sleep "$SLEEP" \ --summary "enrich-${comp}-${src}.md"; then @@ -88,7 +91,8 @@ jobs: # broken while validate, the static dump and the refresh PR were being # skipped every week. Give collection a budget it cannot exceed and # leave the remainder to the rest of the pipeline. - BUDGET_S=$(( 200 * 60 )) + BUDGET_S=$(( 150 * 60 )) + PER_SOURCE_S=$(( 40 * 60 )) DEADLINE=$(( $(date +%s) + BUDGET_S )) budget_left() { if [ "$(date +%s)" -ge "$DEADLINE" ]; then

AltStyle によって変換されたページ (->オリジナル) /