Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 4fafe2d

Browse files
chore: pythonify BK definitions for docker and coverage
We used to manage these definitions in the BK with no versioning or source control, let's move them to python as all the others so that they stay in sync with the latest and greatest updates to the supported platforms and instance types. Signed-off-by: Riccardo Mancini <mancio@amazon.com>
1 parent f101299 commit 4fafe2d

File tree

3 files changed

+85
-8
lines changed

3 files changed

+85
-8
lines changed

‎.buildkite/common.py

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,10 @@ def group(label, command, instances, platforms, **kwargs):
7979
"""
8080
# Use the 1st character of the group name (should be an emoji)
8181
label1 = label[0]
82+
# if the emoji is in the form ":emoji:", pick the entire slug
83+
if label.startswith(':') and ':' in label[1:]:
84+
label1 = label[:label.index(':', 1)+1]
85+
8286
steps = []
8387
commands = command
8488
if isinstance(command, str):
@@ -276,7 +280,7 @@ def __init__(self, with_build_step=True, **kwargs):
276280
if with_build_step:
277281
build_cmds, self.shared_build = shared_build()
278282
self.build_group_per_arch(
279-
"🏗️ Build", build_cmds, depends_on_build=False, set_key=True
283+
"🏗️ Build", build_cmds, depends_on_build=False, set_key=self.shared_build
280284
)
281285
else:
282286
self.shared_build = None
@@ -314,9 +318,25 @@ def _adapt_group(self, group):
314318
for step in group["steps"]:
315319
step["command"] = prepend + step["command"]
316320
if self.shared_build is not None:
317-
step["depends_on"] = self.build_key(
318-
get_arch_for_instance(step["agents"]["instance"])
319-
)
321+
if "depends_on" not in step:
322+
step["depends_on"] = []
323+
elif isinstance(step["depends_on"], str):
324+
step["depends_on"] = [step["depends_on"]]
325+
elif isinstance(step["depends_on"], list):
326+
pass
327+
else:
328+
raise ValueError(
329+
f"depends_on should be a string or a list but is {type(step['depends_on'])}"
330+
)
331+
332+
step["depends_on"].append(self.shared_build)
333+
step["depends_on"] = [
334+
self.build_key(
335+
dep, get_arch_for_instance(step["agents"]["instance"])
336+
)
337+
for dep in step["depends_on"]
338+
]
339+
320340
return group
321341

322342
def build_group(self, *args, **kwargs):
@@ -332,17 +352,17 @@ def build_group(self, *args, **kwargs):
332352
group(*args, **combined), depends_on_build=depends_on_build
333353
)
334354

335-
def build_key(self, arch):
355+
def build_key(self, key, arch):
336356
"""Return the Buildkite key for the build step, for the specified arch"""
337-
return self.shared_build.replace("$(uname -m)", arch).replace(".tar.gz", "")
357+
return key.replace("$(uname -m)", arch).replace(".tar.gz", "")
338358

339359
def build_group_per_arch(self, label, *args, **kwargs):
340360
"""
341361
Build a group, parametrizing over the architectures only.
342362
343363
kwargs consumed by this method and not passed down to `group`:
344364
- `depends_on_build` (default: `True`): Whether the steps in this group depend on the artifacts from the shared compilation steps
345-
- `set_key`: If True, causes the generated steps to have a "key" field
365+
- `set_key`: If a string, causes the generated steps to have a "key" field replacing "$(uname -m)" with arch and removing trailing tar.gz
346366
"""
347367
depends_on_build = kwargs.pop("depends_on_build", True)
348368
set_key = kwargs.pop("set_key", None)
@@ -351,7 +371,7 @@ def build_group_per_arch(self, label, *args, **kwargs):
351371
if set_key:
352372
for step in grp["steps"]:
353373
step["key"] = self.build_key(
354-
get_arch_for_instance(step["agents"]["instance"])
374+
set_key, get_arch_for_instance(step["agents"]["instance"])
355375
)
356376
return self.add_step(grp, depends_on_build=depends_on_build)
357377

‎.buildkite/pipeline_coverage.py

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env python3
2+
# Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
"""Generate Buildkite pipelines dynamically"""
6+
7+
from common import BKPipeline
8+
9+
pipeline = BKPipeline(with_build_step=False)
10+
11+
pipeline.build_group(
12+
":coverage: Coverage",
13+
pipeline.devtool_test(
14+
devtool_opts="--no-build",
15+
pytest_opts="integration_tests/build/test_coverage.py",
16+
),
17+
)
18+
print(pipeline.to_json())

‎.buildkite/pipeline_docker_popular.py

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/usr/bin/env python3
2+
# Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
"""
6+
Buildkite pipeline for testing popular Docker containers
7+
"""
8+
9+
from common import BKPipeline, random_str
10+
11+
pipeline = BKPipeline()
12+
13+
rootfs_tar = f"rootfs_$(uname -m)_{random_str(k=8)}.tar.gz"
14+
15+
pipeline.build_group_per_arch(
16+
":ship: Rootfs build",
17+
[
18+
"sudo yum install -y systemd-container",
19+
"cd tools/test-popular-containers",
20+
"sudo ./build_rootfs.sh",
21+
f'tar czf "{rootfs_tar}" *.ext4',
22+
f'buildkite-agent artifact upload "{rootfs_tar}"',
23+
],
24+
depends_on_build=False,
25+
set_key=rootfs_tar,
26+
)
27+
28+
pipeline.build_group(
29+
":whale: Docker Popular Containers",
30+
[
31+
"./tools/devtool download_ci_artifacts",
32+
f'buildkite-agent artifact download "{rootfs_tar}" .',
33+
f'tar -C tools/test-popular-containers xzf "{rootfs_tar}"',
34+
'./tools/devtool sh "cd ./tools/test-popular-containers; PYTHONPATH=../../tests ./test-docker-rootfs.py"',
35+
],
36+
depends_on=rootfs_tar,
37+
)
38+
39+
print(pipeline.to_json())

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /