Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit cba02e8

Browse files
authored
Improved compile speed by running multi-threaded library discovery. (#2625)
* Simplified error reporting in library detection There is no need to duplicate the preprocessResult/Err variables. This also simplifies naming making it more straighforward. * Remove useless targetFilePath variable * Slight improvement of removeBuildFromSketchFiles * Rename variables for clarity * Removed hardcoded build.warn_data_percentage in build.options file Also fixed the "low memory" warning printer. * Renamed variables for clarity * Renamed variables for clarity * Pre-compute sourceFile fields, and save the in the includes.cache * Added ObjFileIsUpToDate method to sourceFile * Implemented parallel task runner * Simplify use of properties.SplitQuotedString The new release of the library allow ignoring the returned error. arduino/go-properties-orderedmap#42 * Use runner.Task in GCC preprocessor It slightly simplifies code, but also provide the basis for the next commits. * Parallelize library discovery phase in compile * The number of jobs in library detection now follows --jobs flag * Reordered properties construction for clarity * Reordered compileFileWithRecipe for clarity * Added integration test * fix: libraries are recompiled if the list of include paths changes
1 parent 699ddc0 commit cba02e8

File tree

25 files changed

+665
-290
lines changed

25 files changed

+665
-290
lines changed

‎commands/service_compile.go‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu
169169
if buildPathArg := req.GetBuildPath(); buildPathArg != "" {
170170
buildPath = paths.New(req.GetBuildPath()).Canonical()
171171
if in, _ := buildPath.IsInsideDir(sk.FullPath); in && buildPath.IsDir() {
172-
if sk.AdditionalFiles, err = removeBuildFromSketchFiles(sk.AdditionalFiles, buildPath); err != nil {
172+
if sk.AdditionalFiles, err = removeBuildPathFromSketchFiles(sk.AdditionalFiles, buildPath); err != nil {
173173
return err
174174
}
175175
}
@@ -220,10 +220,6 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu
220220
return err
221221
}
222222

223-
actualPlatform := buildPlatform
224-
otherLibrariesDirs := paths.NewPathList(req.GetLibraries()...)
225-
otherLibrariesDirs.Add(s.settings.LibrariesDir())
226-
227223
var libsManager *librariesmanager.LibrariesManager
228224
if profile != nil {
229225
libsManager = lm
@@ -253,6 +249,11 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu
253249
if req.GetVerbose() {
254250
verbosity = logger.VerbosityVerbose
255251
}
252+
253+
librariesDirs := paths.NewPathList(req.GetLibraries()...) // Array of collection of libraries directories
254+
librariesDirs.Add(s.settings.LibrariesDir())
255+
libraryDirs := paths.NewPathList(req.GetLibrary()...) // Array of single-library directories
256+
256257
sketchBuilder, err := builder.NewBuilder(
257258
ctx,
258259
sk,
@@ -264,16 +265,16 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu
264265
int(req.GetJobs()),
265266
req.GetBuildProperties(),
266267
s.settings.HardwareDirectories(),
267-
otherLibrariesDirs,
268+
librariesDirs,
268269
s.settings.IDEBuiltinLibrariesDir(),
269270
fqbn,
270271
req.GetClean(),
271272
req.GetSourceOverride(),
272273
req.GetCreateCompilationDatabaseOnly(),
273-
targetPlatform, actualPlatform,
274+
targetPlatform, buildPlatform,
274275
req.GetSkipLibrariesDiscovery(),
275276
libsManager,
276-
paths.NewPathList(req.GetLibrary()...),
277+
libraryDirs,
277278
outStream, errStream, verbosity, req.GetWarnings(),
278279
progressCB,
279280
pme.GetEnvVarsForSpawnedProcess(),
@@ -462,15 +463,15 @@ func maybePurgeBuildCache(compilationsBeforePurge uint, cacheTTL time.Duration)
462463
buildcache.New(paths.TempDir().Join("arduino", "sketches")).Purge(cacheTTL)
463464
}
464465

465-
// removeBuildFromSketchFiles removes the files contained in the build directory from
466+
// removeBuildPathFromSketchFiles removes the files contained in the build directory from
466467
// the list of the sketch files
467-
func removeBuildFromSketchFiles(files paths.PathList, build *paths.Path) (paths.PathList, error) {
468+
func removeBuildPathFromSketchFiles(files paths.PathList, build *paths.Path) (paths.PathList, error) {
468469
var res paths.PathList
469470
ignored := false
470471
for _, file := range files {
471472
if isInside, _ := file.IsInsideDir(build); !isInside {
472-
res=append(res, file)
473-
} else if!ignored{
473+
res.Add(file)
474+
} else {
474475
ignored = true
475476
}
476477
}

‎commands/service_monitor.go‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ import (
2727
"github.com/arduino/arduino-cli/internal/arduino/cores"
2828
"github.com/arduino/arduino-cli/internal/arduino/cores/packagemanager"
2929
pluggableMonitor "github.com/arduino/arduino-cli/internal/arduino/monitor"
30-
"github.com/arduino/arduino-cli/internal/i18n"
3130
"github.com/arduino/arduino-cli/pkg/fqbn"
3231
rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
3332
"github.com/arduino/go-properties-orderedmap"
@@ -265,10 +264,7 @@ func findMonitorAndSettingsForProtocolAndBoard(pme *packagemanager.Explorer, pro
265264
} else if recipe, ok := boardPlatform.MonitorsDevRecipes[protocol]; ok {
266265
// If we have a recipe we must resolve it
267266
cmdLine := boardProperties.ExpandPropsInString(recipe)
268-
cmdArgs, err := properties.SplitQuotedString(cmdLine, `"'`, false)
269-
if err != nil {
270-
return nil, nil, &cmderrors.InvalidArgumentError{Message: i18n.Tr("Invalid recipe in platform.txt"), Cause: err}
271-
}
267+
cmdArgs, _ := properties.SplitQuotedString(cmdLine, `"'`, false)
272268
id := fmt.Sprintf("%s-%s", boardPlatform, protocol)
273269
return pluggableMonitor.New(id, cmdArgs...), boardSettings, nil
274270
}

‎commands/service_upload.go‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -720,10 +720,7 @@ func runTool(ctx context.Context, recipeID string, props *properties.Map, outStr
720720
return errors.New(i18n.Tr("no upload port provided"))
721721
}
722722
cmdLine := props.ExpandPropsInString(recipe)
723-
cmdArgs, err := properties.SplitQuotedString(cmdLine, `"'`, false)
724-
if err != nil {
725-
return errors.New(i18n.Tr("invalid recipe '%[1]s': %[2]s", recipe, err))
726-
}
723+
cmdArgs, _ := properties.SplitQuotedString(cmdLine, `"'`, false)
727724

728725
// Run Tool
729726
logrus.WithField("phase", "upload").Tracef("Executing upload tool: %s", cmdLine)

‎internal/arduino/builder/builder.go‎

Lines changed: 41 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,6 @@ type Builder struct {
5959
// Parallel processes
6060
jobs int
6161

62-
// Custom build properties defined by user (line by line as "key=value" pairs)
63-
customBuildProperties []string
64-
6562
// core related
6663
coreBuildCachePath *paths.Path
6764
extraCoreBuildCachePaths paths.PathList
@@ -89,7 +86,7 @@ type Builder struct {
8986
lineOffset int
9087

9188
targetPlatform *cores.PlatformRelease
92-
actualPlatform *cores.PlatformRelease
89+
buildPlatform *cores.PlatformRelease
9390

9491
buildArtifacts *buildArtifacts
9592

@@ -125,19 +122,20 @@ func NewBuilder(
125122
coreBuildCachePath *paths.Path,
126123
extraCoreBuildCachePaths paths.PathList,
127124
jobs int,
128-
requestBuildProperties []string,
129-
hardwareDirs, otherLibrariesDirs paths.PathList,
125+
customBuildProperties []string,
126+
hardwareDirs paths.PathList,
127+
librariesDirs paths.PathList,
130128
builtInLibrariesDirs *paths.Path,
131129
fqbn *fqbn.FQBN,
132130
clean bool,
133131
sourceOverrides map[string]string,
134132
onlyUpdateCompilationDatabase bool,
135-
targetPlatform, actualPlatform *cores.PlatformRelease,
133+
targetPlatform, buildPlatform *cores.PlatformRelease,
136134
useCachedLibrariesResolution bool,
137135
librariesManager *librariesmanager.LibrariesManager,
138-
libraryDirs paths.PathList,
136+
customLibraryDirs paths.PathList,
139137
stdout, stderr io.Writer, verbosity logger.Verbosity, warningsLevel string,
140-
progresCB rpc.TaskProgressCB,
138+
progressCB rpc.TaskProgressCB,
141139
toolEnv []string,
142140
) (*Builder, error) {
143141
buildProperties := properties.NewMap()
@@ -146,14 +144,12 @@ func NewBuilder(
146144
}
147145
if sk != nil {
148146
buildProperties.SetPath("sketch_path", sk.FullPath)
147+
buildProperties.Set("build.project_name", sk.MainFile.Base())
148+
buildProperties.SetPath("build.source.path", sk.FullPath)
149149
}
150150
if buildPath != nil {
151151
buildProperties.SetPath("build.path", buildPath)
152152
}
153-
if sk != nil {
154-
buildProperties.Set("build.project_name", sk.MainFile.Base())
155-
buildProperties.SetPath("build.source.path", sk.FullPath)
156-
}
157153
if optimizeForDebug {
158154
if debugFlags, ok := buildProperties.GetOk("compiler.optimization_flags.debug"); ok {
159155
buildProperties.Set("compiler.optimization_flags", debugFlags)
@@ -165,12 +161,11 @@ func NewBuilder(
165161
}
166162

167163
// Add user provided custom build properties
168-
customBuildProperties, err := properties.LoadFromSlice(requestBuildProperties)
169-
if err != nil {
164+
if p, err := properties.LoadFromSlice(customBuildProperties); err == nil {
165+
buildProperties.Merge(p)
166+
} else {
170167
return nil, fmt.Errorf("invalid build properties: %w", err)
171168
}
172-
buildProperties.Merge(customBuildProperties)
173-
customBuildPropertiesArgs := append(requestBuildProperties, "build.warn_data_percentage=75")
174169

175170
sketchBuildPath, err := buildPath.Join("sketch").Abs()
176171
if err != nil {
@@ -190,16 +185,20 @@ func NewBuilder(
190185
}
191186

192187
log := logger.New(stdout, stderr, verbosity, warningsLevel)
193-
libsManager, libsResolver, verboseOut, err := detector.LibrariesLoader(
194-
useCachedLibrariesResolution, librariesManager,
195-
builtInLibrariesDirs, libraryDirs, otherLibrariesDirs,
196-
actualPlatform, targetPlatform,
188+
libsManager, libsResolver, libsLoadingWarnings, err := detector.LibrariesLoader(
189+
useCachedLibrariesResolution,
190+
librariesManager,
191+
builtInLibrariesDirs,
192+
customLibraryDirs,
193+
librariesDirs,
194+
buildPlatform,
195+
targetPlatform,
197196
)
198197
if err != nil {
199198
return nil, err
200199
}
201200
if log.VerbosityLevel() == logger.VerbosityVerbose {
202-
log.Warn(string(verboseOut))
201+
log.Warn(string(libsLoadingWarnings))
203202
}
204203

205204
diagnosticStore := diagnostics.NewStore()
@@ -212,25 +211,26 @@ func NewBuilder(
212211
coreBuildPath: coreBuildPath,
213212
librariesBuildPath: librariesBuildPath,
214213
jobs: jobs,
215-
customBuildProperties: customBuildPropertiesArgs,
216214
coreBuildCachePath: coreBuildCachePath,
217215
extraCoreBuildCachePaths: extraCoreBuildCachePaths,
218216
logger: log,
219217
clean: clean,
220218
sourceOverrides: sourceOverrides,
221219
onlyUpdateCompilationDatabase: onlyUpdateCompilationDatabase,
222220
compilationDatabase: compilation.NewDatabase(buildPath.Join("compile_commands.json")),
223-
Progress: progress.New(progresCB),
221+
Progress: progress.New(progressCB),
224222
executableSectionsSize: []ExecutableSectionSize{},
225223
buildArtifacts: &buildArtifacts{},
226224
targetPlatform: targetPlatform,
227-
actualPlatform: actualPlatform,
225+
buildPlatform: buildPlatform,
228226
toolEnv: toolEnv,
229227
buildOptions: newBuildOptions(
230-
hardwareDirs, otherLibrariesDirs,
231-
builtInLibrariesDirs, buildPath,
228+
hardwareDirs,
229+
librariesDirs,
230+
builtInLibrariesDirs,
231+
buildPath,
232232
sk,
233-
customBuildPropertiesArgs,
233+
customBuildProperties,
234234
fqbn,
235235
clean,
236236
buildProperties.Get("compiler.optimization_flags"),
@@ -322,10 +322,19 @@ func (b *Builder) preprocess() error {
322322
b.librariesBuildPath,
323323
b.buildProperties,
324324
b.targetPlatform.Platform.Architecture,
325+
b.jobs,
325326
)
326327
if err != nil {
327328
return err
328329
}
330+
if b.libsDetector.IncludeFoldersChanged() && b.librariesBuildPath.Exist() {
331+
if b.logger.VerbosityLevel() == logger.VerbosityVerbose {
332+
b.logger.Info(i18n.Tr("The list of included libraries has been changed... rebuilding all libraries."))
333+
}
334+
if err := b.librariesBuildPath.RemoveAll(); err != nil {
335+
return err
336+
}
337+
}
329338
b.Progress.CompleteStep()
330339

331340
b.warnAboutArchIncompatibleLibraries(b.libsDetector.ImportedLibraries())
@@ -492,29 +501,26 @@ func (b *Builder) prepareCommandForRecipe(buildProperties *properties.Map, recip
492501
commandLine = properties.DeleteUnexpandedPropsFromString(commandLine)
493502
}
494503

495-
parts, err := properties.SplitQuotedString(commandLine, `"'`, false)
496-
if err != nil {
497-
return nil, err
498-
}
504+
args, _ := properties.SplitQuotedString(commandLine, `"'`, false)
499505

500506
// if the overall commandline is too long for the platform
501507
// try reducing the length by making the filenames relative
502508
// and changing working directory to build.path
503509
var relativePath string
504510
if len(commandLine) > 30000 {
505511
relativePath = buildProperties.Get("build.path")
506-
for i, arg := range parts {
512+
for i, arg := range args {
507513
if _, err := os.Stat(arg); os.IsNotExist(err) {
508514
continue
509515
}
510516
rel, err := filepath.Rel(relativePath, arg)
511517
if err == nil && !strings.Contains(rel, "..") && len(rel) < len(arg) {
512-
parts[i] = rel
518+
args[i] = rel
513519
}
514520
}
515521
}
516522

517-
command, err := paths.NewProcess(b.toolEnv, parts...)
523+
command, err := paths.NewProcess(b.toolEnv, args...)
518524
if err != nil {
519525
return nil, err
520526
}

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /