Repository: incubator-mynewt-newt Updated Branches: refs/heads/develop 205009dff -> c1dfedcd1
MYNEWT-537 newt - Multithreaded compiles To specify the number of concurrent threads, use the "-j" option: newt -j 5 build bleprph-nrf52dk newt -j 5 create-image bleprph-nrf52dk 1.2.3.4 newt -j 5 run bleprph-nrf52dk 0 Project: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/commit/c1dfedcd Tree: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/tree/c1dfedcd Diff: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/diff/c1dfedcd Branch: refs/heads/develop Commit: c1dfedcd1595882ddd20f042a16fc2ed89260b46 Parents: 205009d Author: Christopher Collins <[email protected]> Authored: Wed Jan 4 12:44:46 2017 -0800 Committer: Christopher Collins <[email protected]> Committed: Wed Jan 4 19:18:05 2017 -0800 ---------------------------------------------------------------------- newt/builder/build.go | 187 +++++++++++++++++++----- newt/cli/run_cmds.go | 2 +- newt/newt.go | 5 + newt/newtutil/newtutil.go | 1 + newt/toolchain/compiler.go | 311 +++++++++++++++++++++------------------- 5 files changed, 321 insertions(+), 185 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/c1dfedcd/newt/builder/build.go ---------------------------------------------------------------------- diff --git a/newt/builder/build.go b/newt/builder/build.go index 6e2fbdb..b41d69d 100644 --- a/newt/builder/build.go +++ b/newt/builder/build.go @@ -27,6 +27,7 @@ import ( log "github.com/Sirupsen/logrus" "mynewt.apache.org/newt/newt/image" + "mynewt.apache.org/newt/newt/newtutil" "mynewt.apache.org/newt/newt/pkg" "mynewt.apache.org/newt/newt/repo" "mynewt.apache.org/newt/newt/symbol" @@ -144,12 +145,12 @@ func pkgTypeConflictErr(p1 *BuildPackage, p2 *BuildPackage) error { // Recursively compiles all the .c and .s files in the specified directory. // Architecture-specific files are also compiled. -func buildDir(srcDir string, c *toolchain.Compiler, arch string, - ignDirs []string) error { +func collectCompileEntriesDir(srcDir string, c *toolchain.Compiler, + arch string, ignDirs []string) ([]toolchain.CompilerJob, error) { // Quietly succeed if the source directory doesn't exist. if util.NodeNotExist(srcDir) { - return nil + return nil, nil } util.StatusMessage(util.VERBOSITY_VERBOSE, @@ -161,23 +162,32 @@ func buildDir(srcDir string, c *toolchain.Compiler, arch string, // Ignore architecture-specific source files for now. Use a temporary // string slice here so that the "arch" directory is not ignored in the // subsequent architecture-specific compile phase. - if err := c.RecursiveCompile(toolchain.COMPILER_TYPE_C, - append(ignDirs, "arch")); err != nil { + ignDirsArch := append(ignDirs, "arch") - return err + entries := []toolchain.CompilerJob{} + + subEntries, err := c.RecursiveCollectEntries(toolchain.COMPILER_TYPE_C, + ignDirsArch) + if err != nil { + return nil, err } + entries = append(entries, subEntries...) // Compile CPP files - if err := c.RecursiveCompile(toolchain.COMPILER_TYPE_CPP, - append(ignDirs, "arch")); err != nil { - return err + subEntries, err = c.RecursiveCollectEntries(toolchain.COMPILER_TYPE_CPP, + ignDirsArch) + if err != nil { + return nil, err } + entries = append(entries, subEntries...) // Copy in pre-compiled library files - if err := c.RecursiveCompile(toolchain.COMPILER_TYPE_ARCHIVE, - append(ignDirs, "arch")); err != nil { - return err + subEntries, err = c.RecursiveCollectEntries( + toolchain.COMPILER_TYPE_ARCHIVE, ignDirsArch) + if err != nil { + return nil, err } + entries = append(entries, subEntries...) archDir := srcDir + "/arch/" + arch + "/" if util.NodeExist(archDir) { @@ -187,33 +197,39 @@ func buildDir(srcDir string, c *toolchain.Compiler, arch string, c.SetSrcDir(archDir) // Compile C source. - if err := c.RecursiveCompile(toolchain.COMPILER_TYPE_C, - ignDirs); err != nil { - - return err + subEntries, err = c.RecursiveCollectEntries( + toolchain.COMPILER_TYPE_C, ignDirs) + if err != nil { + return nil, err } + entries = append(entries, subEntries...) // Compile CPP source - if err := c.RecursiveCompile(toolchain.COMPILER_TYPE_CPP, - ignDirs); err != nil { - return err + subEntries, err = c.RecursiveCollectEntries( + toolchain.COMPILER_TYPE_CPP, ignDirs) + if err != nil { + return nil, err } + entries = append(entries, subEntries...) // Compile assembly source (only architecture-specific). - if err := c.RecursiveCompile(toolchain.COMPILER_TYPE_ASM, - ignDirs); err != nil { - - return err + subEntries, err = c.RecursiveCollectEntries( + toolchain.COMPILER_TYPE_ASM, ignDirs) + if err != nil { + return nil, err } + entries = append(entries, subEntries...) // Copy in pre-compiled library files - if err := c.RecursiveCompile(toolchain.COMPILER_TYPE_ARCHIVE, - ignDirs); err != nil { - return err + subEntries, err = c.RecursiveCollectEntries( + toolchain.COMPILER_TYPE_ARCHIVE, ignDirs) + if err != nil { + return nil, err } + entries = append(entries, subEntries...) } - return nil + return entries, nil } func (b *Builder) newCompiler(bpkg *BuildPackage, @@ -238,11 +254,12 @@ func (b *Builder) newCompiler(bpkg *BuildPackage, return c, nil } -// Compiles and archives a package. -func (b *Builder) buildPackage(bpkg *BuildPackage) error { +func (b *Builder) collectCompileEntriesBpkg(bpkg *BuildPackage) ( + []toolchain.CompilerJob, error) { + c, err := b.newCompiler(bpkg, b.PkgBinDir(bpkg)) if err != nil { - return err + return nil, err } srcDirs := []string{} @@ -251,7 +268,7 @@ func (b *Builder) buildPackage(bpkg *BuildPackage) error { for _, relDir := range bpkg.SourceDirectories { dir := bpkg.BasePath() + "/" + relDir if util.NodeNotExist(dir) { - return util.NewNewtError(fmt.Sprintf( + return nil, util.NewNewtError(fmt.Sprintf( "Specified source directory %s, does not exist.", dir)) } @@ -261,22 +278,33 @@ func (b *Builder) buildPackage(bpkg *BuildPackage) error { srcDir := bpkg.BasePath() + "/src" if util.NodeNotExist(srcDir) { // Nothing to compile. - return nil + return nil, nil } srcDirs = append(srcDirs, srcDir) } + entries := []toolchain.CompilerJob{} for _, dir := range srcDirs { - if err = buildDir(dir, c, b.targetBuilder.bspPkg.Arch, nil); err != nil { - return err + subEntries, err := collectCompileEntriesDir(dir, c, + b.targetBuilder.bspPkg.Arch, nil) + if err != nil { + return nil, err } + + entries = append(entries, subEntries...) } + return entries, nil +} + +func (b *Builder) createArchive(c *toolchain.Compiler, + bpkg *BuildPackage) error { + // Create a static library ("archive"). c.SetSrcDir(bpkg.RelativePath()) archiveFile := b.ArchivePath(bpkg) - if err = c.CompileArchive(archiveFile); err != nil { + if err := c.CompileArchive(archiveFile); err != nil { return err } @@ -427,16 +455,103 @@ func (b *Builder) addSysinitBpkg() (*BuildPackage, error) { return b.addPackage(lpkg) } +// Runs build jobs while any remain. On failure, signals the other workers to +// stop via the stop channel. On error, the error object is signaled via the +// results channel. On successful completion, nil is signaled via the results +// channel. +func buildWorker( + id int, + jobs <-chan toolchain.CompilerJob, + stop chan struct{}, + results chan error) { + + // Execute each job until failure or until a stop is signalled. + for { + select { + case s := <-stop: + // Re-enqueue the stop signal for the other routines. + stop <- s + + // Terminate this go routine. + results <- nil + return + + case j := <-jobs: + if err := toolchain.RunJob(j); err != nil { + // Stop the other routines. + stop <- struct{}{} + + // Report the error back to the master thread and terminate. + results <- err + return + } + + default: + // Terminate this go routine. + results <- nil + return + } + } +} + func (b *Builder) Build() error { b.CleanArtifacts() // Build the packages alphabetically to ensure a consistent order. bpkgs := b.sortedBuildPackages() + // Calculate the list of jobs. Each record represents a single file that + // needs to be compiled. + entries := []toolchain.CompilerJob{} + bpkgCompilerMap := map[*BuildPackage]*toolchain.Compiler{} for _, bpkg := range bpkgs { - if err := b.buildPackage(bpkg); err != nil { + subEntries, err := b.collectCompileEntriesBpkg(bpkg) + if err != nil { return err } + entries = append(entries, subEntries...) + + if len(subEntries) > 0 { + bpkgCompilerMap[bpkg] = subEntries[0].Compiler + } + } + + // Build each file in parallel. + jobs := make(chan toolchain.CompilerJob, len(entries)) + defer close(jobs) + + stop := make(chan struct{}, 1) + defer close(stop) + + errors := make(chan error, newtutil.NewtNumJobs) + defer close(errors) + + for _, entry := range entries { + jobs <- entry + } + + for i := 0; i < newtutil.NewtNumJobs; i++ { + go buildWorker(i, jobs, stop, errors) + } + + var err error + for i := 0; i < newtutil.NewtNumJobs; i++ { + subErr := <-errors + if err == nil && subErr != nil { + err = subErr + } + } + if err != nil { + return err + } + + for _, bpkg := range bpkgs { + c := bpkgCompilerMap[bpkg] + if c != nil { + if err := b.createArchive(c, bpkg); err != nil { + return err + } + } } return nil http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/c1dfedcd/newt/cli/run_cmds.go ---------------------------------------------------------------------- diff --git a/newt/cli/run_cmds.go b/newt/cli/run_cmds.go index 2265d6c..55d7bbc 100644 --- a/newt/cli/run_cmds.go +++ b/newt/cli/run_cmds.go @@ -98,7 +98,7 @@ func AddRunCommands(cmd *cobra.Command) { Run: runRunCmd, } - runCmd.PersistentFlags().StringVarP(&extraJtagCmd, "extrajtagcmd", "j", "", + runCmd.PersistentFlags().StringVarP(&extraJtagCmd, "extrajtagcmd", "", "", "extra commands to send to JTAG software") runCmd.PersistentFlags().BoolVarP(&noGDB_flag, "noGDB", "n", false, "don't start GDB from command line") http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/c1dfedcd/newt/newt.go ---------------------------------------------------------------------- diff --git a/newt/newt.go b/newt/newt.go index e7a4ab0..8395db6 100644 --- a/newt/newt.go +++ b/newt/newt.go @@ -36,6 +36,7 @@ var newtSilent bool var newtQuiet bool var newtVerbose bool var newtLogFile string +var newtNumJobs int func newtCmd() *cobra.Command { newtHelpText := cli.FormatHelp(`Newt allows you to create your own embedded @@ -79,6 +80,8 @@ func newtCmd() *cobra.Command { if err != nil { cli.NewtUsage(nil, err) } + + newtutil.NewtNumJobs = newtNumJobs }, Run: func(cmd *cobra.Command, args []string) { cmd.Help() @@ -95,6 +98,8 @@ func newtCmd() *cobra.Command { "WARN", "Log level") newtCmd.PersistentFlags().StringVarP(&newtLogFile, "outfile", "o", "", "Filename to tee output to") + newtCmd.PersistentFlags().IntVarP(&newtNumJobs, "jobs", "j", + 1, "Number of concurrent build jobs") versHelpText := cli.FormatHelp(`Display the Newt version number.`) versHelpEx := " newt version" http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/c1dfedcd/newt/newtutil/newtutil.go ---------------------------------------------------------------------- diff --git a/newt/newtutil/newtutil.go b/newt/newtutil/newtutil.go index 0182514..824e3af 100644 --- a/newt/newtutil/newtutil.go +++ b/newt/newtutil/newtutil.go @@ -36,6 +36,7 @@ import ( var NewtVersionStr string = "Apache Newt (incubating) version: 1.0.0-dev" var NewtBlinkyTag string = "develop" +var NewtNumJobs int const NEWTRC_DIR string = ".newt" const REPOS_FILENAME string = "repos.yml" http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/c1dfedcd/newt/toolchain/compiler.go ---------------------------------------------------------------------- diff --git a/newt/toolchain/compiler.go b/newt/toolchain/compiler.go index 710243b..e7d962e 100644 --- a/newt/toolchain/compiler.go +++ b/newt/toolchain/compiler.go @@ -30,6 +30,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" log "github.com/Sirupsen/logrus" @@ -60,9 +61,13 @@ type CompilerInfo struct { } type Compiler struct { - ObjPathList map[string]bool + objPathList map[string]bool LinkerScripts []string + // Needs to be locked whenever a mutable field in this struct is accessed + // during a build. Currently, objPathList is the only such member. + mutex *sync.Mutex + depTracker DepTracker ccPath string cppPath string @@ -93,6 +98,12 @@ type Compiler struct { extraDeps []string } +type CompilerJob struct { + Filename string + Compiler *Compiler + CompilerType int +} + func NewCompilerInfo() *CompilerInfo { ci := &CompilerInfo{} ci.Includes = []string{} @@ -177,7 +188,8 @@ func NewCompiler(compilerDir string, dstDir string, buildProfile string) (*Compiler, error) { c := &Compiler{ - ObjPathList: map[string]bool{}, + mutex: &sync.Mutex{}, + objPathList: map[string]bool{}, baseDir: project.GetProject().BasePath, srcDir: "", dstDir: dstDir, @@ -292,7 +304,10 @@ func (c *Compiler) AddDeps(depFilenames ...string) { // executable. func (c *Compiler) SkipSourceFile(srcFile string) error { objPath := c.dstFilePath(srcFile) + ".o" - c.ObjPathList[filepath.ToSlash(objPath)] = true + + c.mutex.Lock() + c.objPathList[filepath.ToSlash(objPath)] = true + c.mutex.Unlock() // Update the dependency tracker with the object file's modification time. // This is necessary later for determining if the library / executable @@ -316,7 +331,7 @@ func (c *Compiler) includesStrings() []string { tokens := make([]string, len(includes)) for i, s := range includes { - s = strings.TrimPrefix(filepath.ToSlash(filepath.Clean(s)), c.baseDir + "/") + s = strings.TrimPrefix(filepath.ToSlash(filepath.Clean(s)), c.baseDir+"/") tokens[i] = "-I" + s } @@ -344,7 +359,7 @@ func (c *Compiler) depsString() string { } func (c *Compiler) dstFilePath(srcPath string) string { - relSrcPath := strings.TrimPrefix(filepath.ToSlash(srcPath), c.srcDir + "/") + relSrcPath := strings.TrimPrefix(filepath.ToSlash(srcPath), c.srcDir+"/") relDstPath := strings.TrimSuffix(relSrcPath, filepath.Ext(srcPath)) dstPath := fmt.Sprintf("%s/%s", c.dstDir, relDstPath) return dstPath @@ -382,7 +397,7 @@ func (c *Compiler) CompileFileCmd(file string, compilerType int) ( return nil, util.NewNewtError("Unknown compiler type") } - srcPath := strings.TrimPrefix(file, c.baseDir + "/") + srcPath := strings.TrimPrefix(file, c.baseDir+"/") cmd := []string{cmdName} cmd = append(cmd, flags...) cmd = append(cmd, c.includesStrings()...) @@ -406,7 +421,7 @@ func (c *Compiler) GenDepsForFile(file string) error { os.MkdirAll(depDir, 0755) } - srcPath := strings.TrimPrefix(file, c.baseDir + "/") + srcPath := strings.TrimPrefix(file, c.baseDir+"/") cmd := []string{c.ccPath} cmd = append(cmd, c.cflagsStrings()...) cmd = append(cmd, c.includesStrings()...) @@ -483,14 +498,16 @@ func (c *Compiler) CompileFile(file string, compilerType int) error { os.MkdirAll(objDir, 0755) } - c.ObjPathList[filepath.ToSlash(objPath)] = true + c.mutex.Lock() + c.objPathList[filepath.ToSlash(objPath)] = true + c.mutex.Unlock() cmd, err := c.CompileFileCmd(file, compilerType) if err != nil { return err } - srcPath := strings.TrimPrefix(file, c.baseDir + "/") + srcPath := strings.TrimPrefix(file, c.baseDir+"/") switch compilerType { case COMPILER_TYPE_C: util.StatusMessage(util.VERBOSITY_DEFAULT, "Compiling %s\n", srcPath) @@ -528,79 +545,71 @@ func (c *Compiler) shouldIgnoreFile(file string) bool { return false } -// Compiles all C files matching the specified file glob. -func (c *Compiler) CompileC() error { - files, _ := filepath.Glob(c.srcDir + "/*.c") - - log.Infof("Compiling C if outdated (%s/*.c) %s", c.srcDir, - strings.Join(files, " ")) +func compilerTypeToExts(compilerType int) ([]string, error) { + switch compilerType { + case COMPILER_TYPE_C: + return []string{"c"}, nil + case COMPILER_TYPE_ASM: + return []string{"s", "S"}, nil + case COMPILER_TYPE_CPP: + return []string{"cc", "cpp", "cxx"}, nil + case COMPILER_TYPE_ARCHIVE: + return []string{"a"}, nil + default: + return nil, util.NewNewtError("Wrong compiler type specified to " + + "compilerTypeToExts") + } +} - for _, file := range files { - file = filepath.ToSlash(file) +// Compiles all C files matching the specified file glob. +func (c *Compiler) CompileC(filename string) error { + filename = filepath.ToSlash(filename) - if shouldIgnore := c.shouldIgnoreFile(file); shouldIgnore { - log.Infof("Ignoring %s because package dictates it.", file) - continue - } + if c.shouldIgnoreFile(filename) { + log.Infof("Ignoring %s because package dictates it.", filename) + return nil + } - compileRequired, err := c.depTracker.CompileRequired(file, - COMPILER_TYPE_C) - if err != nil { - return err - } - if compileRequired { - err = c.CompileFile(file, COMPILER_TYPE_C) - } else { - err = c.SkipSourceFile(file) - } - if err != nil { - return err - } + compileRequired, err := c.depTracker.CompileRequired(filename, + COMPILER_TYPE_C) + if err != nil { + return err + } + if compileRequired { + err = c.CompileFile(filename, COMPILER_TYPE_C) + } else { + err = c.SkipSourceFile(filename) + } + if err != nil { + return err } return nil } // Compiles all CPP files -func (c *Compiler) CompileCpp() error { - files, _ := filepath.Glob(c.srcDir + "/*.cc") - moreFiles, _ := filepath.Glob(c.srcDir + "/*.cpp") - files = append(files, moreFiles...) - moreFiles, _ = filepath.Glob(c.srcDir + "/*.cxx") - files = append(files, moreFiles...) - - wd, err := os.Getwd() +func (c *Compiler) CompileCpp(filename string) error { + filename = filepath.ToSlash(filename) + + if c.shouldIgnoreFile(filename) { + log.Infof("Ignoring %s because package dictates it.", filename) + return nil + } + + compileRequired, err := c.depTracker.CompileRequired(filename, + COMPILER_TYPE_CPP) if err != nil { return err } - wd = filepath.ToSlash(filepath.Clean(wd)) - log.Infof("Working in dir (%s)", wd) - - log.Infof("Compiling CC if outdated (%s/*.cc) %s", wd, - strings.Join(files, " ")) - - for _, file := range files { - file = filepath.ToSlash(file) - - if shouldIgnore := c.shouldIgnoreFile(file); shouldIgnore { - log.Infof("Ignoring %s because package dictates it.", file) - } - compileRequired, err := c.depTracker.CompileRequired(file, - COMPILER_TYPE_CPP) - if err != nil { - return err - } - - if compileRequired { - err = c.CompileFile(file, COMPILER_TYPE_CPP) - } else { - err = c.SkipSourceFile(file) - } + if compileRequired { + err = c.CompileFile(filename, COMPILER_TYPE_CPP) + } else { + err = c.SkipSourceFile(filename) + } - if err != nil { - return err - } + if err != nil { + return err } return nil @@ -610,41 +619,26 @@ func (c *Compiler) CompileCpp() error { // // @param match The file glob specifying which assembly files // to compile. -func (c *Compiler) CompileAs() error { - files, _ := filepath.Glob(c.srcDir + "/*.s") - moreFiles, _ := filepath.Glob(c.srcDir + "/*.S") - files = append(files, moreFiles...) +func (c *Compiler) CompileAs(filename string) error { + filename = filepath.ToSlash(filename) + + if c.shouldIgnoreFile(filename) { + log.Infof("Ignoring %s because package dictates it.", filename) + return nil + } - wd, err := os.Getwd() + compileRequired, err := c.depTracker.CompileRequired(filename, + COMPILER_TYPE_ASM) if err != nil { return err } - wd = filepath.ToSlash(filepath.Clean(wd)) - log.Infof("Working in dir (%s)", wd) - - log.Infof("Compiling assembly if outdated (%s/*.s) %s", wd, - strings.Join(files, " ")) - for _, file := range files { - file = filepath.ToSlash(file) - - if shouldIgnore := c.shouldIgnoreFile(file); shouldIgnore { - log.Infof("Ignoring %s because package dictates it.", file) - continue - } - - compileRequired, err := c.depTracker.CompileRequired(file, - COMPILER_TYPE_ASM) - if err != nil { - return err - } - if compileRequired { - err = c.CompileFile(file, COMPILER_TYPE_ASM) - } else { - err = c.SkipSourceFile(file) - } - if err != nil { - return err - } + if compileRequired { + err = c.CompileFile(filename, COMPILER_TYPE_ASM) + } else { + err = c.SkipSourceFile(filename) + } + if err != nil { + return err } return nil @@ -654,52 +648,46 @@ func (c *Compiler) CompileAs() error { // // @param match The file glob specifying which assembly files // to compile. -func (c *Compiler) CopyArchive() error { - files, _ := filepath.Glob(c.srcDir + "/*.a") - - log.Infof("Copying archive if outdated (%s/*.a) %s", c.srcDir, - strings.Join(files, " ")) - for _, file := range files { - file = filepath.ToSlash(file) +func (c *Compiler) CopyArchive(filename string) error { + filename = filepath.ToSlash(filename) - if shouldIgnore := c.shouldIgnoreFile(file); shouldIgnore { - log.Infof("Ignoring %s because package dictates it.", file) - continue - } + if c.shouldIgnoreFile(filename) { + log.Infof("Ignoring %s because package dictates it.", filename) + return nil + } - tgtFile := c.dstFilePath(file) + ".a" - copyRequired, err := c.depTracker.CopyRequired(file) - if err != nil { - return err - } - if copyRequired { - err = util.CopyFile(file, tgtFile) - util.StatusMessage(util.VERBOSITY_DEFAULT, "copying %s\n", - filepath.ToSlash(tgtFile)) - } + tgtFile := c.dstFilePath(filename) + ".a" + copyRequired, err := c.depTracker.CopyRequired(filename) + if err != nil { + return err + } + if copyRequired { + err = util.CopyFile(filename, tgtFile) + util.StatusMessage(util.VERBOSITY_DEFAULT, "copying %s\n", + filepath.ToSlash(tgtFile)) + } - if err != nil { - return err - } + if err != nil { + return err } return nil } func (c *Compiler) processEntry(node os.FileInfo, cType int, - ignDirs []string) error { + ignDirs []string) ([]CompilerJob, error) { // check to see if we ignore this element - for _, entry := range ignDirs { - if entry == node.Name() { - return nil + for _, dir := range ignDirs { + if dir == node.Name() { + return nil, nil } } // Check in the user specified ignore directories - for _, entry := range c.info.IgnoreDirs { - if entry.MatchString(node.Name()) { - return nil + for _, dir := range c.info.IgnoreDirs { + if dir.MatchString(node.Name()) { + return nil, nil } } @@ -711,61 +699,88 @@ func (c *Compiler) processEntry(node os.FileInfo, cType int, c.srcDir += "/" + node.Name() c.dstDir += "/" + node.Name() - err := c.RecursiveCompile(cType, ignDirs) + entries, err := c.RecursiveCollectEntries(cType, ignDirs) - // Restore the compiler destination directory now that the child directory - // has been fully built. + // Restore the compiler destination directory now that the child + // directory has been fully built. c.srcDir = prevSrcDir c.dstDir = prevDstDir - return err + return entries, err } -func (c *Compiler) RecursiveCompile(cType int, ignDirs []string) error { +func (c *Compiler) RecursiveCollectEntries(cType int, + ignDirs []string) ([]CompilerJob, error) { + // Make sure the compiler package info is added to the global set. c.ensureLclInfoAdded() if err := os.Chdir(c.baseDir); err != nil { - return util.ChildNewtError(err) + return nil, util.ChildNewtError(err) } // Get a list of files in the current directory, and if they are a // directory, and that directory is not in the ignDirs variable, then // recurse into that directory and compile the files in there - dirList, err := ioutil.ReadDir(c.srcDir) + ls, err := ioutil.ReadDir(c.srcDir) if err != nil { - return util.NewNewtError(err.Error()) + return nil, util.NewNewtError(err.Error()) } - for _, node := range dirList { + entries := []CompilerJob{} + for _, node := range ls { if node.IsDir() { - err = c.processEntry(node, cType, ignDirs) + subEntries, err := c.processEntry(node, cType, ignDirs) if err != nil { - return err + return nil, err } + + entries = append(entries, subEntries...) } } - switch cType { + exts, err := compilerTypeToExts(cType) + if err != nil { + return nil, err + } + + for _, ext := range exts { + files, _ := filepath.Glob(c.srcDir + "/*." + ext) + for _, file := range files { + entries = append(entries, CompilerJob{ + Filename: file, + Compiler: c, + CompilerType: cType, + }) + } + } + + return entries, nil +} + +func RunJob(record CompilerJob) error { + switch record.CompilerType { case COMPILER_TYPE_C: - return c.CompileC() + return record.Compiler.CompileC(record.Filename) case COMPILER_TYPE_ASM: - return c.CompileAs() + return record.Compiler.CompileAs(record.Filename) case COMPILER_TYPE_CPP: - return c.CompileCpp() + return record.Compiler.CompileCpp(record.Filename) case COMPILER_TYPE_ARCHIVE: - return c.CopyArchive() + return record.Compiler.CopyArchive(record.Filename) default: return util.NewNewtError("Wrong compiler type specified to " + - "RecursiveCompile") + "RunJob") } } func (c *Compiler) getObjFiles(baseObjFiles []string) []string { - for objName, _ := range c.ObjPathList { + c.mutex.Lock() + for objName, _ := range c.objPathList { baseObjFiles = append(baseObjFiles, objName) } + c.mutex.Unlock() sort.Strings(baseObjFiles) return baseObjFiles
