Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 52eefd8

Browse files
chore: pythonify BK definitions for docker and coverage
We used to manage these definitions in the BK with no versioning or source control, let's move them to python as all the others so that they stay in sync with the latest and greatest updates to the supported platforms and instance types. Signed-off-by: Riccardo Mancini <mancio@amazon.com>
1 parent 0b2fd57 commit 52eefd8

File tree

4 files changed

+86
-8
lines changed

4 files changed

+86
-8
lines changed

‎.buildkite/common.py

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,10 @@ def group(label, command, instances, platforms, **kwargs):
7878
"""
7979
# Use the 1st character of the group name (should be an emoji)
8080
label1 = label[0]
81+
# if the emoji is in the form ":emoji:", pick the entire slug
82+
if label.startswith(":") and ":" in label[1:]:
83+
label1 = label[: label.index(":", 1) + 1]
84+
8185
steps = []
8286
commands = command
8387
if isinstance(command, str):
@@ -275,7 +279,7 @@ def __init__(self, with_build_step=True, **kwargs):
275279
if with_build_step:
276280
build_cmds, self.shared_build = shared_build()
277281
self.build_group_per_arch(
278-
"🏗️ Build", build_cmds, depends_on_build=False, set_key=True
282+
"🏗️ Build", build_cmds, depends_on_build=False, set_key=self.shared_build
279283
)
280284
else:
281285
self.shared_build = None
@@ -313,9 +317,25 @@ def _adapt_group(self, group):
313317
for step in group["steps"]:
314318
step["command"] = prepend + step["command"]
315319
if self.shared_build is not None:
316-
step["depends_on"] = self.build_key(
317-
get_arch_for_instance(step["agents"]["instance"])
318-
)
320+
if "depends_on" not in step:
321+
step["depends_on"] = []
322+
elif isinstance(step["depends_on"], str):
323+
step["depends_on"] = [step["depends_on"]]
324+
elif isinstance(step["depends_on"], list):
325+
pass
326+
else:
327+
raise ValueError(
328+
f"depends_on should be a string or a list but is {type(step['depends_on'])}"
329+
)
330+
331+
step["depends_on"].append(self.shared_build)
332+
step["depends_on"] = [
333+
self.build_key(
334+
dep, get_arch_for_instance(step["agents"]["instance"])
335+
)
336+
for dep in step["depends_on"]
337+
]
338+
319339
return group
320340

321341
def build_group(self, *args, **kwargs):
@@ -331,17 +351,17 @@ def build_group(self, *args, **kwargs):
331351
group(*args, **combined), depends_on_build=depends_on_build
332352
)
333353

334-
def build_key(self, arch):
354+
def build_key(self, key, arch):
335355
"""Return the Buildkite key for the build step, for the specified arch"""
336-
return self.shared_build.replace("$(uname -m)", arch).replace(".tar.gz", "")
356+
return key.replace("$(uname -m)", arch).replace(".tar.gz", "")
337357

338358
def build_group_per_arch(self, label, *args, **kwargs):
339359
"""
340360
Build a group, parametrizing over the architectures only.
341361
342362
kwargs consumed by this method and not passed down to `group`:
343363
- `depends_on_build` (default: `True`): Whether the steps in this group depend on the artifacts from the shared compilation steps
344-
- `set_key`: If True, causes the generated steps to have a "key" field
364+
- `set_key`: If a string, causes the generated steps to have a "key" field replacing "$(uname -m)" with arch and removing trailing tar.gz
345365
"""
346366
depends_on_build = kwargs.pop("depends_on_build", True)
347367
set_key = kwargs.pop("set_key", None)
@@ -350,7 +370,7 @@ def build_group_per_arch(self, label, *args, **kwargs):
350370
if set_key:
351371
for step in grp["steps"]:
352372
step["key"] = self.build_key(
353-
get_arch_for_instance(step["agents"]["instance"])
373+
set_key, get_arch_for_instance(step["agents"]["instance"])
354374
)
355375
return self.add_step(grp, depends_on_build=depends_on_build)
356376

‎.buildkite/pipeline_coverage.py

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env python3
2+
# Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
"""Generate Buildkite pipelines dynamically"""
6+
7+
from common import BKPipeline
8+
9+
pipeline = BKPipeline(with_build_step=False)
10+
11+
pipeline.build_group(
12+
":coverage: Coverage",
13+
pipeline.devtool_test(
14+
devtool_opts="--no-build",
15+
pytest_opts="integration_tests/build/test_coverage.py",
16+
),
17+
)
18+
print(pipeline.to_json())

‎.buildkite/pipeline_docker_popular.py

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/usr/bin/env python3
2+
# Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
"""
6+
Buildkite pipeline for testing popular Docker containers
7+
"""
8+
9+
from common import BKPipeline, random_str
10+
11+
pipeline = BKPipeline()
12+
13+
ROOTFS_TAR = f"rootfs_$(uname -m)_{random_str(k=8)}.tar.gz"
14+
15+
pipeline.build_group_per_arch(
16+
":ship: Rootfs build",
17+
[
18+
"sudo yum install -y systemd-container",
19+
"cd tools/test-popular-containers",
20+
"sudo ./build_rootfs.sh",
21+
f'tar czf "{ROOTFS_TAR}" *.ext4 *.id_rsa',
22+
f'buildkite-agent artifact upload "{ROOTFS_TAR}"',
23+
],
24+
depends_on_build=False,
25+
set_key=ROOTFS_TAR,
26+
)
27+
28+
pipeline.build_group(
29+
":whale: Docker Popular Containers",
30+
[
31+
"./tools/devtool download_ci_artifacts",
32+
f'buildkite-agent artifact download "{ROOTFS_TAR}" .',
33+
f'tar xzf "{ROOTFS_TAR}" -C tools/test-popular-containers',
34+
'./tools/devtool sh "cd ./tools/test-popular-containers; PYTHONPATH=../../tests ./test-docker-rootfs.py"',
35+
],
36+
depends_on=ROOTFS_TAR,
37+
)
38+
39+
print(pipeline.to_json())

‎tools/test-popular-containers/build_rootfs.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ function make_rootfs {
2424
ssh-keygen -f id_rsa -N ""
2525
fi
2626
cp id_rsa $rootfs.id_rsa
27+
chmod a+r $rootfs.id_rsa
2728

2829
truncate -s "$SIZE" "$IMG"
2930
mkfs.ext4 -F "$IMG" -d $LABEL

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /