Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit ec74a8b

Browse files
Merge pull request #9230 from Neon-White/warp-helpers
Add Warp-specific IBM Cloud VM nightly runs tooling
2 parents bc4b42e + 76f0d8f commit ec74a8b

File tree

7 files changed

+135
-6
lines changed

7 files changed

+135
-6
lines changed

‎.github/ibm-warp-runner-config.yaml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#cloud-config
2+
package_update: true
3+
packages: [docker.io, git, make, unzip, jq, curl, ca-certificates, nodejs, npm]
4+
5+
write_files:
6+
- path: /etc/warp.env
7+
permissions: '0600'
8+
content: |
9+
NODE_PATH=/usr/local/lib/node_modules
10+
SLACK_NIGHTLY_RESULTS_URL=${SLACK_NIGHTLY_RESULTS_URL}
11+
AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
12+
AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
13+
WARP_LOGS_BUCKET=${WARP_LOGS_BUCKET}
14+
IBM_COS_ENDPOINT=${IBM_COS_ENDPOINT}
15+
16+
runcmd:
17+
# Schedule VM shutdown after 4 hours (240 minutes) as a safety measure
18+
- [ shutdown, -P, '+240' ]
19+
# Add ubuntu user to docker group for container access
20+
- [ usermod, -aG, docker, ubuntu ]
21+
# Enable and start Docker daemon
22+
- [ systemctl, enable, --now, docker ]
23+
# Install Slack webhook package globally for the notifier script
24+
- [ npm, -g, install, '@slack/webhook' ]
25+
# Download, extract and install AWS CLI v2
26+
- [ curl, -sS, https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip, -o, awscliv2.zip ]
27+
- [ unzip, awscliv2.zip ]
28+
- [ ./aws/install ]
29+
# Create tests directory with proper ownership and permissions
30+
- [ install, -d, -m, '755', -o, ubuntu, -g, ubuntu, /home/ubuntu/tests ]
31+
# Clone NooBaa core repository for testing
32+
- [ git, clone, https://github.com/noobaa/noobaa-core.git, /home/ubuntu/tests/noobaa-core ]
33+
# Install warp runner script to system path with proper permissions
34+
- [ install, -m, '755', -o, ubuntu, -g, ubuntu, /home/ubuntu/tests/noobaa-core/tools/ibm_runner_helpers/run_containerized_warp_on_cloud_runner.sh, /usr/local/bin/ ]
35+
# Install Slack notifier script to system path with proper permissions
36+
- [ install, -m, '755', -o, ubuntu, -g, ubuntu, /home/ubuntu/tests/noobaa-core/tools/ibm_runner_helpers/slack_notifier.js, /usr/local/bin/ ]
37+
# Execute the main warp testing script (this will run the tests and handle results)
38+
- [ /usr/local/bin/run_containerized_warp_on_cloud_runner.sh ]

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -357,15 +357,15 @@ test-warp: tester
357357
@$(call create_docker_network)
358358
@$(call run_postgres)
359359
@echo "033円[1;34mRunning warp tests033円[0m"
360-
$(CONTAINER_ENGINE) run $(CPUSET) --privileged --user root --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "POSTGRES_HOST=coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "POSTGRES_USER=noobaa" --env "DB_TYPE=postgres" --env "POSTGRES_DBNAME=coretest" -v $(PWD)/logs:/logs $(TESTER_TAG) "./src/test/external_tests/warp/run_warp_on_test_container.sh"
360+
$(CONTAINER_ENGINE) run $(CPUSET) --privileged --user root --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "POSTGRES_HOST=coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "POSTGRES_USER=noobaa" --env "DB_TYPE=postgres" --env "POSTGRES_DBNAME=coretest" -v $(PWD)/logs:/logs $(TESTER_TAG) bash -c "./src/test/external_tests/warp/run_warp_on_test_container.sh$(WARP_ARGS)"
361361
@$(call stop_noobaa)
362362
@$(call stop_postgres)
363363
@$(call remove_docker_network)
364364
.PHONY: test-warp
365365

366366
test-nc-warp: tester
367367
@echo "033円[1;34mRunning warp tests on NC environment033円[0m"
368-
$(CONTAINER_ENGINE) run $(CPUSET) --privileged --user root --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" -v $(PWD)/logs:/logs $(TESTER_TAG) "./src/test/external_tests/warp/run_nc_warp_on_test_container.sh"
368+
$(CONTAINER_ENGINE) run $(CPUSET) --privileged --user root --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" -v $(PWD)/logs:/logs $(TESTER_TAG) bash -c "./src/test/external_tests/warp/run_nc_warp_on_test_container.sh$(WARP_ARGS)"
369369
.PHONY: test-nc-warp
370370

371371
test-mint: tester

‎src/test/external_tests/warp/run_nc_warp_on_test_container.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,6 @@ node ./src/test/external_tests/warp/configure_warp.js
7070
# ====================================================================================
7171

7272
# Run the warp tests
73-
node ./src/test/external_tests/warp/run_warp.js
73+
node ./src/test/external_tests/warp/run_warp.js"$@"
7474

7575
# ====================================================================================

‎src/test/external_tests/warp/run_warp.js‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,11 @@ Usage:
2828
set the access key to use for the tests. by default it is set to $access_key.
2929
--secret-key <string>
3030
set the secret key to use for the tests. by default it is set to $secret_key.
31-
--obj-size <string>
31+
--obj-size <string>
3232
set the object size to use for the tests. by default it is set to 1k.
3333
example: 1k, 1m, 1g
34+
--obj-randsize <boolean>
35+
set to true to enable random object sizes. by default it is set to false.
3436
--account-name <string>
3537
set the account name to use for the tests. by default it is set to warp_account.
3638
`;
@@ -73,6 +75,7 @@ async function run_warp() {
7375
const duration = argv.duration || '10m';
7476
const number_of_workers = argv.concurrency || DEFAULT_NUMBER_OF_WORKERS;
7577
const disable_multipart = argv.disable_multipart || true;
78+
const obj_randsize = argv.obj_randsize || false;
7679
const endpoint = config.ENDPOINT_SSL_PORT;
7780

7881
if (!account_name && !access_key && !secret_key) {
@@ -86,7 +89,7 @@ async function run_warp() {
8689

8790
// TODO - add --benchdata so that the result csv will be saved in logs
8891
// const warp_logs_dir = WARP_TEST.warp_logs_dir_path;
89-
const warp_command = `warp ${op} --host=localhost:${endpoint} --access-key=${access_key} --secret-key=${secret_key} --bucket=${bucket} --obj.size=${obj_size} --duration=${duration} --disable-multipart=${disable_multipart} --tls --insecure --concurrent ${number_of_workers}`;
92+
const warp_command = `warp ${op} --host=localhost:${endpoint} --access-key=${access_key} --secret-key=${secret_key} --bucket=${bucket} --obj.size=${obj_size} --duration=${duration} --disable-multipart=${disable_multipart} --tls --insecure --concurrent ${number_of_workers}${obj_randsize ? ' --obj.randsize' : ''}`;
9093

9194
console.info(`Running warp ${warp_command}`);
9295
try {

‎src/test/external_tests/warp/run_warp_on_test_container.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,6 @@ sleep 90
7070
# ====================================================================================
7171

7272
# Run the warp tests
73-
node ./src/test/external_tests/warp/run_warp.js
73+
node ./src/test/external_tests/warp/run_warp.js"$@"
7474

7575
# ====================================================================================
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/usr/bin/env bash
2+
set -euxo pipefail
3+
set -a
4+
. /etc/warp.env
5+
set +a
6+
PS4='+ [run_warp] '
7+
export HOME=/home/ubuntu
8+
9+
# Global variables for log directory naming
10+
timestamp=$(date +"%Y-%m-%d-%H-%M-%S")
11+
log_dir="warp-test-logs-${timestamp}"
12+
test_logs_path="/home/ubuntu/tests/noobaa-core/logs/warp-test-logs"
13+
14+
send_failure_notification() {
15+
local msg="1ドル"
16+
node /usr/local/bin/slack_notifier.js "${SLACK_NIGHTLY_RESULTS_URL}" failure "Warp failed: ${msg}" || true
17+
}
18+
19+
send_success_notification() {
20+
local log_location="1ドル"
21+
node /usr/local/bin/slack_notifier.js "${SLACK_NIGHTLY_RESULTS_URL}" success "Warp success - logs available at: ${log_location}" || true
22+
}
23+
24+
handle_success() {
25+
echo "Uploading logs to IBM COS s3://${WARP_LOGS_BUCKET}/${log_dir}/"
26+
aws s3 cp "${test_logs_path}" "s3://${WARP_LOGS_BUCKET}/${log_dir}/" --recursive --no-progress --endpoint-url "${IBM_COS_ENDPOINT}"
27+
echo "Successfully uploaded logs to IBM COS s3://${WARP_LOGS_BUCKET}/${log_dir}/"
28+
send_success_notification "s3://${WARP_LOGS_BUCKET}/${log_dir}/"
29+
}
30+
31+
handle_failure() {
32+
echo "Uploading cloud-init logs to IBM COS s3://${WARP_LOGS_BUCKET}/${log_dir}/"
33+
aws s3 cp "/var/log/cloud-init-output.log" "s3://${WARP_LOGS_BUCKET}/${log_dir}/" --no-progress --endpoint-url "${IBM_COS_ENDPOINT}"
34+
echo "Successfully uploaded cloud-init logs to IBM COS s3://${WARP_LOGS_BUCKET}/${log_dir}/"
35+
send_failure_notification "Run failed, logs available at s3://${WARP_LOGS_BUCKET}/${log_dir}/"
36+
}
37+
38+
shutdown_vm() {
39+
echo "Shutting down VM..."
40+
sudo shutdown -P now
41+
}
42+
43+
trap 'handle_failure "line $LINENO exit $?"' ERR
44+
trap 'shutdown_vm' EXIT
45+
46+
cd /home/ubuntu/tests/noobaa-core
47+
48+
echo "Building NooBaa images locally..."
49+
make tester
50+
51+
echo "Creating Warp logs directory..."
52+
mkdir -p logs/warp-test-logs
53+
chmod 755 logs/warp-test-logs
54+
55+
echo "Running Warp tests..."
56+
make test-warp -o tester WARP_ARGS="--duration 3h --obj-size 10m --obj-randsize"
57+
58+
handle_success
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/* Copyright (C) 2025 NooBaa */
2+
'use strict';
3+
4+
const { IncomingWebhook } = require('@slack/webhook');
5+
6+
async function main() {
7+
const [,, webhookUrl, status, message] = process.argv;
8+
if (!webhookUrl || !status || !message) {
9+
console.error('Usage: node slack_notifier.js <webhook_url> <status> <message>');
10+
process.exit(0);
11+
}
12+
13+
try {
14+
const webhook = new IncomingWebhook(webhookUrl);
15+
const emoji = status === 'success' ? '✅' : '❌';
16+
const text = emoji + ' ' + message;
17+
18+
// Send message only
19+
await webhook.send({
20+
text: text
21+
});
22+
23+
} catch (error) {
24+
console.error('Notification failed:', error.message);
25+
}
26+
27+
process.exit(0);
28+
}
29+
30+
main();

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /