Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 652bcac

Browse files
cmaglieumbynos
andauthored
Allow downloading index and signature in a single tar.bz2 archive (#1734)
* Allow downloading index and signature in a tar.bz2 archive * Make some configurations resilient to empty settings * Added tests * fix comment * Apply suggestions from code review Co-authored-by: Umberto Baldi <34278123+umbynos@users.noreply.github.com> * Allow .tar.bz2 package index URL * Fix errors in timeout checks * Fixed check for stale package_index.json files Co-authored-by: Umberto Baldi <34278123+umbynos@users.noreply.github.com>
1 parent b7721c8 commit 652bcac

File tree

12 files changed

+114
-20
lines changed

12 files changed

+114
-20
lines changed

‎arduino/cores/packagemanager/package_manager.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,11 @@ func (pm *PackageManager) ResolveFQBN(fqbn *cores.FQBN) (
237237

238238
// LoadPackageIndex loads a package index by looking up the local cached file from the specified URL
239239
func (pm *PackageManager) LoadPackageIndex(URL *url.URL) error {
240-
indexPath := pm.IndexDir.Join(path.Base(URL.Path))
240+
indexFileName := path.Base(URL.Path)
241+
if strings.HasSuffix(indexFileName, ".tar.bz2") {
242+
indexFileName = strings.TrimSuffix(indexFileName, ".tar.bz2") + ".json"
243+
}
244+
indexPath := pm.IndexDir.Join(indexFileName)
241245
index, err := packageindex.LoadIndex(indexPath)
242246
if err != nil {
243247
return fmt.Errorf(tr("loading json index file %[1]s: %[2]s"), indexPath, err)

‎arduino/resources/index.go‎

Lines changed: 46 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
package resources
1717

1818
import (
19+
"context"
1920
"net/url"
2021
"path"
2122
"strings"
@@ -25,6 +26,8 @@ import (
2526
"github.com/arduino/arduino-cli/arduino/security"
2627
rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
2728
"github.com/arduino/go-paths-helper"
29+
"github.com/codeclysm/extract/v3"
30+
"github.com/sirupsen/logrus"
2831
"go.bug.st/downloader/v2"
2932
)
3033

@@ -56,8 +59,43 @@ func (res *IndexResource) Download(destDir *paths.Path, downloadCB rpc.DownloadP
5659
return &arduino.FailedDownloadError{Message: tr("Error downloading index '%s'", res.URL), Cause: err}
5760
}
5861

62+
var signaturePath, tmpSignaturePath *paths.Path
63+
hasSignature := false
64+
5965
// Expand the index if it is compressed
60-
if strings.HasSuffix(indexFileName, ".gz") {
66+
if strings.HasSuffix(indexFileName, ".tar.bz2") {
67+
indexFileName = strings.TrimSuffix(indexFileName, ".tar.bz2") + ".json" // == package_index.json
68+
signatureFileName := indexFileName + ".sig"
69+
signaturePath = destDir.Join(signatureFileName)
70+
71+
// .tar.bz2 archive may contain both index and signature
72+
73+
// Extract archive in a tmp/archive subdirectory
74+
f, err := tmpIndexPath.Open()
75+
if err != nil {
76+
return &arduino.PermissionDeniedError{Message: tr("Error opening %s", tmpIndexPath), Cause: err}
77+
}
78+
defer f.Close()
79+
tmpArchivePath := tmp.Join("archive")
80+
_ = tmpArchivePath.MkdirAll()
81+
if err := extract.Bz2(context.Background(), f, tmpArchivePath.String(), nil); err != nil {
82+
return &arduino.PermissionDeniedError{Message: tr("Error extracting %s", tmpIndexPath), Cause: err}
83+
}
84+
85+
// Look for index.json
86+
tmpIndexPath = tmpArchivePath.Join(indexFileName)
87+
if !tmpIndexPath.Exist() {
88+
return &arduino.NotFoundError{Message: tr("Invalid archive: file %{1}s not found in archive %{2}s", indexFileName, tmpArchivePath.Base())}
89+
}
90+
91+
// Look for signature
92+
if t := tmpArchivePath.Join(signatureFileName); t.Exist() {
93+
tmpSignaturePath = t
94+
hasSignature = true
95+
} else {
96+
logrus.Infof("No signature %s found in package index archive %s", signatureFileName, tmpArchivePath.Base())
97+
}
98+
} else if strings.HasSuffix(indexFileName, ".gz") {
6199
indexFileName = strings.TrimSuffix(indexFileName, ".gz") // == package_index.json
62100
tmpUnzippedIndexPath := tmp.Join(indexFileName)
63101
if err := paths.GUnzip(tmpIndexPath, tmpUnzippedIndexPath); err != nil {
@@ -67,7 +105,6 @@ func (res *IndexResource) Download(destDir *paths.Path, downloadCB rpc.DownloadP
67105
}
68106

69107
// Check the signature if needed
70-
var signaturePath, tmpSignaturePath *paths.Path
71108
if res.SignatureURL != nil {
72109
// Compose signature URL
73110
signatureFileName := path.Base(res.SignatureURL.Path)
@@ -79,6 +116,10 @@ func (res *IndexResource) Download(destDir *paths.Path, downloadCB rpc.DownloadP
79116
return &arduino.FailedDownloadError{Message: tr("Error downloading index signature '%s'", res.SignatureURL), Cause: err}
80117
}
81118

119+
hasSignature = true
120+
}
121+
122+
if hasSignature {
82123
// Check signature...
83124
if valid, _, err := security.VerifyArduinoDetachedSignature(tmpIndexPath, tmpSignaturePath); err != nil {
84125
return &arduino.PermissionDeniedError{Message: tr("Error verifying signature"), Cause: err}
@@ -109,12 +150,12 @@ func (res *IndexResource) Download(destDir *paths.Path, downloadCB rpc.DownloadP
109150
if err := tmpIndexPath.CopyTo(indexPath); err != nil {
110151
return &arduino.PermissionDeniedError{Message: tr("Error saving downloaded index"), Cause: err}
111152
}
112-
if res.SignatureURL!=nil {
153+
if hasSignature {
113154
if err := tmpSignaturePath.CopyTo(signaturePath); err != nil {
114155
return &arduino.PermissionDeniedError{Message: tr("Error saving downloaded index signature"), Cause: err}
115156
}
116157
}
117-
oldIndex.Remove()
118-
oldSignature.Remove()
158+
_=oldIndex.Remove()
159+
_=oldSignature.Remove()
119160
return nil
120161
}

‎arduino/resources/resources_test.go‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@ package resources
1818
import (
1919
"crypto"
2020
"encoding/hex"
21+
"net"
22+
"net/http"
23+
"net/url"
2124
"testing"
2225

2326
rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
@@ -110,3 +113,38 @@ func TestDownloadAndChecksums(t *testing.T) {
110113
_, err = r.TestLocalArchiveChecksum(tmp)
111114
require.Error(t, err)
112115
}
116+
117+
func TestIndexDownloadAndSignatureWithinArchive(t *testing.T) {
118+
// Spawn test webserver
119+
mux := http.NewServeMux()
120+
fs := http.FileServer(http.Dir("testdata"))
121+
mux.Handle("/", fs)
122+
server := &http.Server{Handler: mux}
123+
ln, err := net.Listen("tcp", "127.0.0.1:")
124+
require.NoError(t, err)
125+
defer ln.Close()
126+
go server.Serve(ln)
127+
128+
validIdxURL, err := url.Parse("http://" + ln.Addr().String() + "/valid/package_index.tar.bz2")
129+
require.NoError(t, err)
130+
idxResource := &IndexResource{URL: validIdxURL}
131+
destDir, err := paths.MkTempDir("", "")
132+
require.NoError(t, err)
133+
defer destDir.RemoveAll()
134+
err = idxResource.Download(destDir, func(curr *rpc.DownloadProgress) {})
135+
require.NoError(t, err)
136+
require.True(t, destDir.Join("package_index.json").Exist())
137+
require.True(t, destDir.Join("package_index.json.sig").Exist())
138+
139+
invalidIdxURL, err := url.Parse("http://" + ln.Addr().String() + "/invalid/package_index.tar.bz2")
140+
require.NoError(t, err)
141+
invIdxResource := &IndexResource{URL: invalidIdxURL}
142+
invDestDir, err := paths.MkTempDir("", "")
143+
require.NoError(t, err)
144+
defer invDestDir.RemoveAll()
145+
err = invIdxResource.Download(invDestDir, func(curr *rpc.DownloadProgress) {})
146+
require.Error(t, err)
147+
require.Contains(t, err.Error(), "invalid signature")
148+
require.False(t, invDestDir.Join("package_index.json").Exist())
149+
require.False(t, invDestDir.Join("package_index.json.sig").Exist())
150+
}
39.5 KB
Binary file not shown.
39.5 KB
Binary file not shown.

‎cli/core/search.go‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -134,10 +134,9 @@ func indexesNeedUpdating(duration string) bool {
134134

135135
now := time.Now()
136136
modTimeThreshold, err := time.ParseDuration(duration)
137-
// Not the most elegant way of handling this error
138-
// but it does its job
139137
if err != nil {
140-
modTimeThreshold, _ = time.ParseDuration("24h")
138+
feedback.Error(tr("Invalid timeout: %s", err))
139+
os.Exit(errorcodes.ErrBadArgument)
141140
}
142141

143142
urls := []string{globals.DefaultIndexURL}
@@ -153,7 +152,18 @@ func indexesNeedUpdating(duration string) bool {
153152
continue
154153
}
155154

156-
coreIndexPath := indexpath.Join(path.Base(URL.Path))
155+
// should handle:
156+
// - package_index.json
157+
// - package_index.json.sig
158+
// - package_index.json.gz
159+
// - package_index.tar.bz2
160+
indexFileName := path.Base(URL.Path)
161+
indexFileName = strings.TrimSuffix(indexFileName, ".tar.bz2")
162+
indexFileName = strings.TrimSuffix(indexFileName, ".gz")
163+
indexFileName = strings.TrimSuffix(indexFileName, ".sig")
164+
indexFileName = strings.TrimSuffix(indexFileName, ".json")
165+
// and obtain package_index.json as result
166+
coreIndexPath := indexpath.Join(indexFileName + ".json")
157167
if coreIndexPath.NotExist() {
158168
return true
159169
}

‎cli/globals/globals.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,5 @@ var (
2626
// VersionInfo contains all info injected during build
2727
VersionInfo = version.NewInfo(filepath.Base(os.Args[0]))
2828
// DefaultIndexURL is the default index url
29-
DefaultIndexURL = "https://downloads.arduino.cc/packages/package_index.json"
29+
DefaultIndexURL = "https://downloads.arduino.cc/packages/package_index.tar.bz2"
3030
)

‎commands/instances.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -434,7 +434,7 @@ func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexRequest, downloadCB rp
434434
indexResource := resources.IndexResource{
435435
URL: URL,
436436
}
437-
if strings.HasSuffix(URL.Host, "arduino.cc") {
437+
if strings.HasSuffix(URL.Host, "arduino.cc") &&strings.HasSuffix(URL.Path, ".json") {
438438
indexResource.SignatureURL, _ = url.Parse(u) // should not fail because we already parsed it
439439
indexResource.SignatureURL.Path += ".sig"
440440
}

‎configuration/network.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,10 @@ import (
2626

2727
// UserAgent returns the user agent (mainly used by HTTP clients)
2828
func UserAgent(settings *viper.Viper) string {
29-
subComponent := settings.GetString("network.user_agent_ext")
29+
subComponent := ""
30+
if settings != nil {
31+
subComponent = settings.GetString("network.user_agent_ext")
32+
}
3033
if subComponent != "" {
3134
subComponent = " " + subComponent
3235
}
@@ -41,7 +44,7 @@ func UserAgent(settings *viper.Viper) string {
4144

4245
// NetworkProxy returns the proxy configuration (mainly used by HTTP clients)
4346
func NetworkProxy(settings *viper.Viper) (*url.URL, error) {
44-
if !settings.IsSet("network.proxy") {
47+
if settings==nil||!settings.IsSet("network.proxy") {
4548
return nil, nil
4649
}
4750
if proxyConfig := settings.GetString("network.proxy"); proxyConfig == "" {

‎test/test_board.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
"official": true,
3131
"package": {
3232
"maintainer": "Arduino",
33-
"url": "https://downloads.arduino.cc/packages/package_index.json",
33+
"url": "https://downloads.arduino.cc/packages/package_index.tar.bz2",
3434
"website_url": "http://www.arduino.cc/",
3535
"email": "packages@arduino.cc",
3636
"name": "arduino",

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /