fix(ci): give the dump its own schedule instead of the scrape's leftovers #60

Original file line number	Diff line number	Diff line change
Expand Up		@@ -5,6 +5,14 @@ name: dump-refresh
		# after an engine-side change that affects serialized output (e.g. a scoring
		# version bump) — weekly-refresh.yml also does this but behind a slow live scrape.
		on:
	# The dump has to be reconciled with data/ on a schedule of its own.
	# weekly-refresh also regenerates it, but only after a multi-hour live
	# scrape, and when that scrape runs long the dump is the step that gets
	# dropped — which is how the published pages drifted away from the
	# records for six weeks. Run a day after the Monday jobs so this picks
	# up whatever they merged.
	schedule:
	- cron: "0 3 * * 2" # Tuesdays 03:00 UTC
		workflow_dispatch:

		permissions:
Expand All		@@ -17,6 +25,9 @@ concurrency:
		jobs:
		dump:
		runs-on: ubuntu-latest
	# Well under GitHub's 6h ceiling, which reports an over-run as
	# "cancelled" rather than failed and so hides the breakage.
	timeout-minutes: 330
		env:
		TECHAPI_WRITE_TOKEN: ${{ secrets.TECHENGINEBOT_TOKEN \|\| secrets.TECHAPI_TOKEN }}
		# seed/validate/dump read the data tree from here.
Expand Down

8 changes: 6 additions & 2 deletions .github/workflows/weekly-refresh.yml

Show comments View file Open in desktop

Original file line number	Diff line number	Diff line change
Expand Up		@@ -73,7 +73,10 @@ jobs:
		run_enrich() {
		comp="1ドル"; src="2ドル"
		echo "::group::enrich ${comp}/${src}"
	if python -m app.ingest.enrich \
	# Cap each source: the budget below is only consulted between
	# sources, so without this one slow source overruns it — the last
	# run spent 260 minutes against a 200-minute budget that way.
	if timeout "${PER_SOURCE_S}s" python -m app.ingest.enrich \
		--source "$src" --component "$comp" \
		--data-root ./techapi/data --sleep "$SLEEP" \
		--summary "enrich-${comp}-${src}.md"; then
Expand All		@@ -88,7 +91,8 @@ jobs:
		# broken while validate, the static dump and the refresh PR were being
		# skipped every week. Give collection a budget it cannot exceed and
		# leave the remainder to the rest of the pipeline.
	BUDGET_S=$(( 200 * 60 ))
	BUDGET_S=$(( 150 * 60 ))
	PER_SOURCE_S=$(( 40 * 60 ))
		DEADLINE=$(( $(date +%s) + BUDGET_S ))
		budget_left() {
		if [ "$(date +%s)" -ge "$DEADLINE" ]; then
Expand Down

Navigation Menu