Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 4917a48

Browse files
author
vinay-lanka
committed
fixup! Added tests
2 parents 96a2df7 + 763b4a7 commit 4917a48

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+687
-274
lines changed

‎.github/tools/fetch_athena_stats.sh‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
#!/usr/bin/env bash
2+
3+
# This script performs the following:
4+
# 1. Run the query, use jq to capture the QueryExecutionId, and then capture that into bash variable
5+
# 2. Wait for the query to finish running (240 seconds).
6+
# 3. Get the results.
7+
# 4. Json data points struct build
8+
9+
# Expected env variables are:
10+
# AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY for accessing AWS resources
11+
# AWS_ATHENA_SOURCE_TABLE
12+
# AWS_ATHENA_OUTPUT_LOCATION
13+
# GITHUB_REPOSITORY
14+
15+
set -euo pipefail
16+
17+
! read -r -d '' query << EOM
18+
select
19+
replace(url_extract_path("d.url"), '/arduino-cli/arduino-cli_', '') as flavor,
20+
count("id") as gauge
21+
from ${AWS_ATHENA_SOURCE_TABLE}
22+
where "d.url" like 'https://downloads.arduino.cc/arduino-cli/arduino-cli_%'
23+
and "d.url" not like '%latest%' -- exclude latest redirect
24+
and "d.url" not like '%alpha%' -- exclude early alpha releases
25+
and "d.url" not like '%.tar.bz2%' -- exclude very old releases archive formats
26+
group by 1
27+
EOM
28+
29+
queryExecutionId=$(
30+
aws athena start-query-execution \
31+
--query-string "${query}" \
32+
--query-execution-context "Database=demo_books" \
33+
--result-configuration "OutputLocation=${AWS_ATHENA_OUTPUT_LOCATION}" \
34+
--region us-east-1 | jq -r ".QueryExecutionId"
35+
)
36+
37+
echo "QueryExecutionId is ${queryExecutionId}"
38+
for i in $(seq 1 120); do
39+
queryState=$( aws athena get-query-execution \
40+
--query-execution-id "${queryExecutionId}" \
41+
--region us-east-1 | jq -r ".QueryExecution.Status.State"
42+
);
43+
44+
if [[ "${queryState}" == "SUCCEEDED" ]]; then
45+
break;
46+
fi;
47+
48+
echo "QueryExecutionId ${queryExecutionId} - state is ${queryState}"
49+
50+
if [[ "${queryState}" == "FAILED" ]]; then
51+
exit 1;
52+
fi;
53+
54+
sleep 2
55+
done
56+
57+
echo "Query succeeded. Processing data"
58+
queryResult=$( aws athena get-query-results \
59+
--query-execution-id "${queryExecutionId}" \
60+
--region us-east-1 | jq --compact-output
61+
);
62+
63+
! read -r -d '' jsonTemplate << EOM
64+
{
65+
"type": "gauge",
66+
"name": "arduino.downloads.total",
67+
"value": "%s",
68+
"host": "${GITHUB_REPOSITORY}",
69+
"tags": [
70+
"version:%s",
71+
"os:%s",
72+
"arch:%s",
73+
"cdn:downloads.arduino.cc",
74+
"project:arduino-cli"
75+
]
76+
},
77+
EOM
78+
79+
datapoints="["
80+
for row in $(echo "${queryResult}" | jq 'del(.ResultSet.Rows[0])' | jq -r '.ResultSet.Rows[] | .Data' --compact-output); do
81+
value=$(jq -r ".[1].VarCharValue" <<< "${row}")
82+
tag=$(jq -r ".[0].VarCharValue" <<< "${row}")
83+
# Some splitting to obtain 0.6.0, Windows, 32bit elements from string 0.6.0_Windows_32bit.zip
84+
split=($(echo "$tag" | tr '_' '\n'))
85+
if [[ ${#split[@]} -ne 3 ]]; then
86+
continue
87+
fi
88+
archSplit=($(echo "${split[2]}" | tr '.' '\n'))
89+
datapoints+=$(printf "${jsonTemplate}" "${value}" "${split[0]}" "${split[1]}" "${archSplit[0]}")
90+
done
91+
datapoints="${datapoints::-1}]"
92+
93+
echo "::set-output name=result::$(jq --compact-output <<< "${datapoints}")"

‎.github/workflows/arduino-stats.yaml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
name: arduino-stats
2+
3+
on:
4+
schedule:
5+
# run every day at 12:30:00
6+
- cron: '30 12 * * *'
7+
8+
jobs:
9+
push-stats:
10+
runs-on: ubuntu-latest
11+
12+
steps:
13+
- name: Checkout
14+
uses: actions/checkout@v1
15+
16+
- name: Fetch downloads count form Arduino CDN using AWS Athena
17+
id: fetch
18+
env:
19+
AWS_ACCESS_KEY_ID: ${{ secrets.STATS_AWS_ACCESS_KEY_ID }}
20+
AWS_SECRET_ACCESS_KEY: ${{ secrets.STATS_AWS_SECRET_ACCESS_KEY }}
21+
AWS_ATHENA_SOURCE_TABLE: ${{ secrets.STATS_AWS_ATHENA_SOURCE_TABLE }}
22+
AWS_ATHENA_OUTPUT_LOCATION: ${{ secrets.STATS_AWS_ATHENA_OUTPUT_LOCATION }}
23+
GITHUB_REPOSITORY: ${{ github.repository }}
24+
run: |
25+
# Fetch jq 1.6 as VM has only 1.5 ATM
26+
wget -q https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -O jq
27+
chmod +x jq
28+
PATH=${{ github.workspace }}:$PATH
29+
.github/tools/fetch_athena_stats.sh
30+
31+
- name: Send metrics
32+
uses: masci/datadog@v1
33+
with:
34+
api-key: ${{ secrets.DD_API_KEY }}
35+
# Metrics input expects YAML but JSON will work just right.
36+
metrics: ${{steps.fetch.outputs.result}}
37+
38+
- name: Report failure
39+
if: failure()
40+
uses: masci/datadog@v1
41+
with:
42+
api-key: ${{ secrets.DD_API_KEY }}
43+
events: |
44+
- title: "Arduino CLI stats failing"
45+
text: "Stats collection failed"
46+
alert_type: "error"
47+
host: ${{ github.repository }}
48+
tags:
49+
- "project:arduino-cli"
50+
- "cdn:downloads.arduino.cc"

‎.github/workflows/github-stats.yaml‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
name: github-stats
2+
3+
on:
4+
schedule:
5+
# run every 30 minutes
6+
- cron: '*/30 * * * *'
7+
8+
jobs:
9+
push-stats:
10+
runs-on: ubuntu-latest
11+
12+
steps:
13+
- name: Fetch downloads count
14+
id: fetch
15+
uses: actions/github-script@0.2.0
16+
with:
17+
github-token: ${{github.token}}
18+
script: |
19+
let metrics = []
20+
21+
// Get a list of releases
22+
const opts = github.repos.listReleases.endpoint.merge({
23+
...context.repo
24+
})
25+
const releases = await github.paginate(opts)
26+
27+
// Get download stats for every release
28+
for (const rel of releases) {
29+
// Names for assets are like `arduino-cli_0.4.0_Linux_32bit.tar.gz`,
30+
// we'll use this later to split the asset file name more easily
31+
const baseName = `arduino-cli_${rel.name}_`
32+
33+
// Get a list of assets for this release
34+
const opts = github.repos.listAssetsForRelease.endpoint.merge({
35+
...context.repo,
36+
release_id: rel.id
37+
})
38+
const assets = await github.paginate(opts)
39+
40+
for (const asset of assets) {
41+
// Ignore files that are not arduino-cli packages
42+
if (!asset.name.startsWith(baseName)) {
43+
continue
44+
}
45+
46+
// Strip the base and remove file extension to get `Linux_32bit`
47+
systemArch = asset.name.replace(baseName, "").split(".")[0].split("_")
48+
49+
// Add a metric object to the list of gathered metrics
50+
metrics.push({
51+
"type": "gauge",
52+
"name": "arduino.downloads.total",
53+
"value": asset.download_count,
54+
"host": "${{ github.repository }}",
55+
"tags": [
56+
`version:${rel.name}`,
57+
`os:${systemArch[0]}`,
58+
`arch:${systemArch[1]}`,
59+
"cdn:github.com",
60+
"project:arduino-cli"
61+
]
62+
})
63+
}
64+
}
65+
66+
// The action will put whatever we return from this function in
67+
// `outputs.result`, JSON encoded. So we just return the array
68+
// of objects and GitHub will do the rest.
69+
return metrics
70+
71+
- name: Send metrics
72+
uses: masci/datadog@v1
73+
with:
74+
api-key: ${{ secrets.DD_API_KEY }}
75+
# Metrics input expects YAML but JSON will work just right.
76+
metrics: ${{steps.fetch.outputs.result}}
77+
78+
- name: Report failure
79+
if: failure()
80+
uses: masci/datadog@v1
81+
with:
82+
api-key: ${{ secrets.DD_API_KEY }}
83+
events: |
84+
- title: "Arduino CLI stats failing"
85+
text: "Stats collection failed"
86+
alert_type: "error"
87+
host: ${{ github.repository }}
88+
tags:
89+
- "project:arduino-cli"
90+
- "cdn:github.com"

‎.github/workflows/nightly.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,16 @@ jobs:
3333
PLUGIN_BUCKET: ${{ secrets.DOWNLOADS_BUCKET }}
3434
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
3535
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
36+
37+
- name: Report failure
38+
if: failure()
39+
uses: masci/datadog@v1
40+
with:
41+
api-key: ${{ secrets.DD_API_KEY }}
42+
events: |
43+
- title: "Arduino CLI nighly build failed"
44+
text: "Nightly build worfklow has failed"
45+
alert_type: "error"
46+
host: ${{ github.repository }}
47+
tags:
48+
- "project:arduino-cli"

‎.github/workflows/release.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: release
33
on:
44
push:
55
tags:
6-
- '[0-9].[0-9].[0-9]*'
6+
- '[0-9]+.[0-9]+.[0-9]+*'
77

88
jobs:
99

‎arduino/cores/packagemanager/download.go‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ package packagemanager
1717

1818
import (
1919
"fmt"
20-
"net/http"
2120

2221
"github.com/arduino/arduino-cli/arduino/cores"
2322
"go.bug.st/downloader"
@@ -101,16 +100,16 @@ func (pm *PackageManager) FindPlatformReleaseDependencies(item *PlatformReferenc
101100

102101
// DownloadToolRelease downloads a ToolRelease. If the tool is already downloaded a nil Downloader
103102
// is returned.
104-
func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, downloaderHeaders http.Header) (*downloader.Downloader, error) {
103+
func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, config*downloader.Config) (*downloader.Downloader, error) {
105104
resource := tool.GetCompatibleFlavour()
106105
if resource == nil {
107106
return nil, fmt.Errorf("tool not available for your OS")
108107
}
109-
return resource.Download(pm.DownloadDir, downloaderHeaders)
108+
return resource.Download(pm.DownloadDir, config)
110109
}
111110

112111
// DownloadPlatformRelease downloads a PlatformRelease. If the platform is already downloaded a
113112
// nil Downloader is returned.
114-
func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, downloaderHeaders http.Header) (*downloader.Downloader, error) {
115-
return platform.Resource.Download(pm.DownloadDir, downloaderHeaders)
113+
func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, config*downloader.Config) (*downloader.Downloader, error) {
114+
return platform.Resource.Download(pm.DownloadDir, config)
116115
}

‎arduino/libraries/librariesmanager/download.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ import (
2525
var LibraryIndexURL, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json")
2626

2727
// UpdateIndex downloads the libraries index file from Arduino repository.
28-
func (lm *LibrariesManager) UpdateIndex() (*downloader.Downloader, error) {
28+
func (lm *LibrariesManager) UpdateIndex(config*downloader.Config) (*downloader.Downloader, error) {
2929
lm.IndexFile.Parent().MkdirAll()
3030
// TODO: Download from gzipped URL index
31-
return downloader.Download(lm.IndexFile.String(), LibraryIndexURL.String(), downloader.NoResume)
31+
return downloader.DownloadWithConfig(lm.IndexFile.String(), LibraryIndexURL.String(), *config, downloader.NoResume)
3232
}

‎arduino/resources/helpers.go‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ package resources
1717

1818
import (
1919
"fmt"
20-
"net/http"
2120
"os"
2221

2322
"github.com/arduino/go-paths-helper"
@@ -44,7 +43,7 @@ func (r *DownloadResource) IsCached(downloadDir *paths.Path) (bool, error) {
4443
}
4544

4645
// Download a DownloadResource.
47-
func (r *DownloadResource) Download(downloadDir *paths.Path, downloaderHeaders http.Header) (*downloader.Downloader, error) {
46+
func (r *DownloadResource) Download(downloadDir *paths.Path, config*downloader.Config) (*downloader.Downloader, error) {
4847
cached, err := r.TestLocalArchiveIntegrity(downloadDir)
4948
if err != nil {
5049
return nil, fmt.Errorf("testing local archive integrity: %s", err)
@@ -72,7 +71,5 @@ func (r *DownloadResource) Download(downloadDir *paths.Path, downloaderHeaders h
7271
return nil, fmt.Errorf("getting archive file info: %s", err)
7372
}
7473

75-
downloadConfig := downloader.Config{
76-
RequestHeaders: downloaderHeaders}
77-
return downloader.DownloadWithConfig(path.String(), r.URL, downloadConfig)
74+
return downloader.DownloadWithConfig(path.String(), r.URL, *config)
7875
}

‎arduino/resources/helpers_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525

2626
"github.com/arduino/go-paths-helper"
2727
"github.com/stretchr/testify/require"
28+
"go.bug.st/downloader"
2829
)
2930

3031
type EchoHandler struct{}
@@ -52,7 +53,7 @@ func TestDownloadApplyUserAgentHeaderUsingConfig(t *testing.T) {
5253
URL: srv.URL,
5354
}
5455

55-
d, err := r.Download(tmp, http.Header{"User-Agent": []string{goldUserAgentValue}})
56+
d, err := r.Download(tmp, &downloader.Config{RequestHeaders: http.Header{"User-Agent": []string{goldUserAgentValue}}})
5657
require.NoError(t, err)
5758
err = d.Run()
5859
require.NoError(t, err)

‎arduino/resources/resources_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,11 @@ package resources
1818
import (
1919
"crypto"
2020
"encoding/hex"
21-
"net/http"
2221
"testing"
2322

2423
"github.com/arduino/go-paths-helper"
2524
"github.com/stretchr/testify/require"
25+
"go.bug.st/downloader"
2626
)
2727

2828
func TestDownloadAndChecksums(t *testing.T) {
@@ -42,7 +42,7 @@ func TestDownloadAndChecksums(t *testing.T) {
4242
require.NoError(t, err)
4343

4444
downloadAndTestChecksum := func() {
45-
d, err := r.Download(tmp, http.Header{})
45+
d, err := r.Download(tmp, &downloader.Config{})
4646
require.NoError(t, err)
4747
err = d.Run()
4848
require.NoError(t, err)
@@ -58,7 +58,7 @@ func TestDownloadAndChecksums(t *testing.T) {
5858
downloadAndTestChecksum()
5959

6060
// Download with cached file
61-
d, err := r.Download(tmp, http.Header{})
61+
d, err := r.Download(tmp, &downloader.Config{})
6262
require.NoError(t, err)
6363
require.Nil(t, d)
6464

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /