From 1ddfea35ee7b8305fe50ee841ff89fbd71707cae Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Sat, 18 Dec 2021 12:27:11 -0500 Subject: [PATCH 1/7] Allocate correct capacity for workspace json globs --- cli/internal/context/context.go | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index 028293bee1346..9ce863a437fbf 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -219,15 +219,9 @@ func WithGraph(rootpath string, config *config.Config) Option { // until all parsing is complete // and populate the graph parseJSONWaitGroup := new(errgroup.Group) - justJsons := make([]string, len(spaces)) - for i, space := range spaces { - justJsons[i] = path.Join(space, "package.json") - } - ignore := []string{ - "**/node_modules/**/*", - "**/bower_components/**/*", - "**/test/**/*", - "**/tests/**/*", + justJsons := make([]string, 0, len(spaces)) + for _, space := range spaces { + justJsons = append(justJsons, path.Join(space, "package.json")) } f := globby.GlobFiles(rootpath, justJsons, ignore) From 9aa193050055bba5bdb55a6a6e88b05c158fde45 Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Sat, 18 Dec 2021 12:29:25 -0500 Subject: [PATCH 2/7] Extract workspace ignores into constant slice --- cli/internal/context/context.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index 9ce863a437fbf..45d9036bfba3f 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -224,7 +224,7 @@ func WithGraph(rootpath string, config *config.Config) Option { justJsons = append(justJsons, path.Join(space, "package.json")) } - f := globby.GlobFiles(rootpath, justJsons, ignore) + f := globby.GlobFiles(rootpath, justJsons, getWorkspaceIgnores()) for i, val := range f { _, val := i, val // https://golang.org/doc/faq#closures_and_goroutines @@ -507,3 +507,12 @@ func safeCompileIgnoreFile(filepath string) (*gitignore.GitIgnore, error) { // no op return gitignore.CompileIgnoreLines([]string{}...), nil } + +func getWorkspaceIgnores() []string { + return []string{ + "**/node_modules/**/*", + "**/bower_components/**/*", + "**/test/**/*", + "**/tests/**/*", + } +} From 8b06380040754d5eb894748d1bb0702ca337bf12 Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Sat, 18 Dec 2021 12:36:06 -0500 Subject: [PATCH 3/7] Better encapsulate ColorCache --- cli/internal/context/color_cache.go | 54 +++++++++++++++++++++++++++++ cli/internal/context/context.go | 33 +----------------- cli/internal/run/run.go | 4 +-- 3 files changed, 57 insertions(+), 34 deletions(-) create mode 100644 cli/internal/context/color_cache.go diff --git a/cli/internal/context/color_cache.go b/cli/internal/context/color_cache.go new file mode 100644 index 0000000000000..e11af384c7022 --- /dev/null +++ b/cli/internal/context/color_cache.go @@ -0,0 +1,54 @@ +package context + +import ( + "sync" + + "github.com/fatih/color" +) + +type colorFn = func(format string, a ...interface{}) string + +func getTerminalPackageColors() []colorFn { + return []colorFn{color.CyanString, color.MagentaString, color.GreenString, color.YellowString, color.BlueString} +} + +type ColorCache struct { + mu sync.Mutex + index int + TermColors []colorFn + Cache map[interface{}]colorFn +} + +func NewColorCache() *ColorCache { + return &ColorCache{ + TermColors: getTerminalPackageColors(), + index: 0, + Cache: make(map[interface{}]colorFn), + } +} + +// PrefixColor returns a color function for a given package name +func (c *ColorCache) PrefixColor(name string) colorFn { + c.mu.Lock() + defer c.mu.Unlock() + colorFn, ok := c.Cache[name] + if ok { + return colorFn + } + c.index++ + colorFn = c.TermColors[positiveMod(c.index, 5)] // 5 possible colors + c.Cache[name] = colorFn + return colorFn +} + +// postitiveMod returns a modulo operator like JavaScripts +func positiveMod(x, d int) int { + x = x % d + if x >= 0 { + return x + } + if d < 0 { + return x - d + } + return x + d +} diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index 45d9036bfba3f..86ba521cee2d4 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -15,7 +15,6 @@ import ( "turbo/internal/util" mapset "github.com/deckarep/golang-set" - "github.com/fatih/color" "github.com/google/chrometracing" "github.com/pyr-sh/dag" gitignore "github.com/sabhiram/go-gitignore" @@ -35,19 +34,6 @@ const ( Pnpm ) -type colorFn = func(format string, a ...interface{}) string - -var ( - childProcessIndex = 0 - terminalPackageColors = [5]colorFn{color.CyanString, color.MagentaString, color.GreenString, color.YellowString, color.BlueString} -) - -type ColorCache struct { - sync.Mutex - index int - Cache map[interface{}]colorFn -} - // Context of the CLI type Context struct { Args []string @@ -86,20 +72,6 @@ func New(opts ...Option) (*Context, error) { return &m, nil } -// PrefixColor returns a color function for a given package name -func PrefixColor(c *Context, name *string) colorFn { - c.ColorCache.Lock() - defer c.ColorCache.Unlock() - colorFn, ok := c.ColorCache.Cache[name] - if ok { - return colorFn - } - c.ColorCache.index++ - colorFn = terminalPackageColors[util.PositiveMod(c.ColorCache.index, len(terminalPackageColors))] - c.ColorCache.Cache[name] = colorFn - return colorFn -} - // WithDir specifies the directory where turbo is initiated func WithDir(d string) Option { return func(m *Context) error { @@ -141,10 +113,7 @@ func WithTracer(filename string) Option { func WithGraph(rootpath string, config *config.Config) Option { return func(c *Context) error { c.PackageInfos = make(map[interface{}]*fs.PackageJSON) - c.ColorCache = &ColorCache{ - Cache: make(map[interface{}]colorFn), - index: 0, - } + c.ColorCache = NewColorCache() c.RootNode = ROOT_NODE_NAME c.PendingTaskNodes = make(dag.Set) // Need to ALWAYS have a root node, might as well do it now diff --git a/cli/internal/run/run.go b/cli/internal/run/run.go index 3b886897a90a8..7ed07781ad9ed 100644 --- a/cli/internal/run/run.go +++ b/cli/internal/run/run.go @@ -391,8 +391,8 @@ func (c *RunCommand) Run(args []string) int { tracer := runState.Run(context.GetTaskId(pack.Name, task)) // Create a logger - pref := context.PrefixColor(ctx, &pack.Name) - actualPrefix := pref("%v:%v: ", pack.Name, task) + pref := ctx.ColorCache.PrefixColor(pack.Name) + actualPrefix := pref("%s:%s: ", pack.Name, task) targetUi := &cli.PrefixedUi{ Ui: c.Ui, OutputPrefix: actualPrefix, From 9cc7b6a692cc08008cd7daf77251fe81fa9d328c Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Sat, 18 Dec 2021 12:51:18 -0500 Subject: [PATCH 4/7] Set ExternalDeps capacity --- cli/internal/context/context.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index 86ba521cee2d4..cae8aa87f04b5 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -300,9 +300,9 @@ func (c *Context) ResolveWorkspaceRootDeps() (*fs.PackageJSON, error) { pkg.SubLockfile = make(fs.YarnLockfile) c.ResolveDepGraph(&lockfileWg, pkg.UnresolvedExternalDeps, depSet, seen, pkg) lockfileWg.Wait() - pkg.ExternalDeps = make([]string, depSet.Cardinality()) - for i, v := range depSet.ToSlice() { - pkg.ExternalDeps[i] = v.(string) + pkg.ExternalDeps = make([]string, 0, depSet.Cardinality()) + for _, v := range depSet.ToSlice() { + pkg.ExternalDeps = append(pkg.ExternalDeps, fmt.Sprintf("%s", v)) } sort.Strings(pkg.ExternalDeps) hashOfExternalDeps, err := fs.HashObject(pkg.ExternalDeps) From ad6a497b4c6deaa96ffcee2e183145da1f9e4f0f Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Sat, 18 Dec 2021 12:53:29 -0500 Subject: [PATCH 5/7] Remove redudant word --- cli/internal/context/context.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index cae8aa87f04b5..7718de6baee1c 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -423,7 +423,7 @@ func (c *Context) parsePackageJSON(buildFilePath string) error { if fs.FileExists(buildFilePath) { pkg, err := fs.ReadPackageJSON(buildFilePath) if err != nil { - return fmt.Errorf("error parsing %v: %w", buildFilePath, err) + return fmt.Errorf("parsing %s: %w", buildFilePath, err) } // log.Printf("[TRACE] adding %+v to graph", pkg.Name) From 39650fa7374ffae40af2835ca9557771130d7e81 Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Sat, 18 Dec 2021 13:11:21 -0500 Subject: [PATCH 6/7] Better slice allocation in dependency analysis --- cli/internal/context/context.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index 7718de6baee1c..2bcb673616339 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -297,7 +297,7 @@ func (c *Context) ResolveWorkspaceRootDeps() (*fs.PackageJSON, error) { pkg.UnresolvedExternalDeps[dep] = version } if c.Backend.Name == "nodejs-yarn" && !fs.CheckIfWindows() { - pkg.SubLockfile = make(fs.YarnLockfile) + pkg.SubLockfile = fs.YarnLockfile{} c.ResolveDepGraph(&lockfileWg, pkg.UnresolvedExternalDeps, depSet, seen, pkg) lockfileWg.Wait() pkg.ExternalDeps = make([]string, 0, depSet.Cardinality()) @@ -397,13 +397,13 @@ func (c *Context) populateTopologicGraphForPackageJson(pkg *fs.PackageJSON) erro if internalDepsSet.Len() == 0 { c.TopologicalGraph.Connect(dag.BasicEdge(pkg.Name, ROOT_NODE_NAME)) } - pkg.ExternalDeps = make([]string, externalDepSet.Cardinality()) - for i, v := range externalDepSet.ToSlice() { - pkg.ExternalDeps[i] = v.(string) + pkg.ExternalDeps = make([]string, 0, externalDepSet.Cardinality()) + for _, v := range externalDepSet.ToSlice() { + pkg.ExternalDeps = append(pkg.ExternalDeps, fmt.Sprintf("%s", v)) } - pkg.InternalDeps = make([]string, internalDepsSet.Len()) - for i, v := range internalDepsSet.List() { - pkg.InternalDeps[i] = v.(string) + pkg.InternalDeps = make([]string, 0, internalDepsSet.Len()) + for _, v := range internalDepsSet.List() { + pkg.ExternalDeps = append(pkg.InternalDeps, fmt.Sprintf("%s", v)) } sort.Strings(pkg.InternalDeps) sort.Strings(pkg.ExternalDeps) From 0181c63621d4935fbb0d4db0aa7e33f7d8e64e0c Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Sat, 18 Dec 2021 13:24:37 -0500 Subject: [PATCH 7/7] Clean up package deps hash --- cli/internal/fs/package_deps_hash.go | 32 +++++++++++++--------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/cli/internal/fs/package_deps_hash.go b/cli/internal/fs/package_deps_hash.go index 23af9681b2d16..0040ee2595365 100644 --- a/cli/internal/fs/package_deps_hash.go +++ b/cli/internal/fs/package_deps_hash.go @@ -7,7 +7,6 @@ import ( "path/filepath" "regexp" "strings" - "turbo/internal/util" ) // Predefine []byte variables to avoid runtime allocations. @@ -34,7 +33,7 @@ type PackageDepsOptions struct { func GetPackageDeps(p *PackageDepsOptions) (map[string]string, error) { gitLsOutput, err := gitLsTree(p.PackagePath, p.GitPath) if err != nil { - return nil, fmt.Errorf("Could not get git hashes for files in package %s: %w", p.PackagePath, err) + return nil, fmt.Errorf("getting git hashes for files in package %s: %w", p.PackagePath, err) } // Add all the checked in hashes. result := parseGitLsTree(gitLsOutput) @@ -53,14 +52,13 @@ func GetPackageDeps(p *PackageDepsOptions) (map[string]string, error) { } currentlyChangedFiles := parseGitStatus(gitStatusOutput, p.PackagePath) var filesToHash []string - excludedPathsSet := new(util.Set) + for filename, changeType := range currentlyChangedFiles { if changeType == "D" || (len(changeType) == 2 && string(changeType)[1] == []byte("D")[0]) { delete(result, filename) } else { - if !excludedPathsSet.Include(filename) { - filesToHash = append(filesToHash, filename) - } + // TODO: handle excluded paths here and only hash if included + filesToHash = append(filesToHash, filename) } } @@ -83,13 +81,15 @@ func GetPackageDeps(p *PackageDepsOptions) (map[string]string, error) { return result, nil } +const hashCommand = "hash-object" + // GitHashForFiles a list of files returns a map of with their git hash values. It uses // git hash-object under the func GitHashForFiles(filesToHash []string, PackagePath string) (map[string]string, error) { - changes := make(map[string]string) + changes := make(map[string]string, len(filesToHash)) if len(filesToHash) > 0 { - var input = []string{"hash-object"} - + input := make([]string, 0, len(filesToHash)+1) + input = append(input, hashCommand) for _, filename := range filesToHash { input = append(input, filepath.Join(PackagePath, filename)) } @@ -105,11 +105,10 @@ func GitHashForFiles(filesToHash []string, PackagePath string) (map[string]strin offByOne := strings.Split(string(out), "\n") // there is an extra "" hashes := offByOne[:len(offByOne)-1] if len(hashes) != len(filesToHash) { - return nil, fmt.Errorf("passed %v file paths to Git to hash, but received %v hashes.", len(filesToHash), len(hashes)) + return nil, fmt.Errorf("passed %v file paths to git to hash, but received %v hashes", len(filesToHash), len(hashes)) } for i, hash := range hashes { - filepath := filesToHash[i] - changes[filepath] = hash + changes[filesToHash[i]] = hash } } @@ -129,12 +128,11 @@ func UnescapeChars(in []byte) []byte { // gitLsTree executes "git ls-tree" in a folder func gitLsTree(path string, gitPath string) (string, error) { - cmd := exec.Command("git", "ls-tree", "HEAD", "-r") cmd.Dir = path out, err := cmd.CombinedOutput() if err != nil { - return "", fmt.Errorf("Failed to read `git ls-tree`: %w", err) + return "", fmt.Errorf("reading `git ls-tree`: %w", err) } return strings.TrimSpace(string(out)), nil } @@ -151,7 +149,7 @@ func parseGitLsTree(output string) map[string]string { for _, line := range outputLines { if len(line) > 0 { matches := gitRex.MatchString(line) - if matches == true { + if matches { // this looks like this // [["160000 commit c5880bf5b0c6c1f2e2c43c95beeb8f0a808e8bac rushstack" "160000" "commit" "c5880bf5b0c6c1f2e2c43c95beeb8f0a808e8bac" "rushstack"]] match := gitRex.FindAllStringSubmatch(line, -1) @@ -218,7 +216,7 @@ func gitStatus(path string, gitPath string) (string, error) { cmd.Dir = path out, err := cmd.CombinedOutput() if err != nil { - return "", fmt.Errorf("Failed to read git status: %w", err) + return "", fmt.Errorf("reading git status: %w", err) } // log.Printf("[TRACE] gitStatus result: %v", strings.TrimSpace(string(out))) return strings.TrimSpace(string(out)), nil @@ -247,7 +245,7 @@ func parseGitStatus(output string, PackagePath string) map[string]string { for _, line := range outputLines { if len(line) > 0 { matches := gitRex.MatchString(line) - if matches == true { + if matches { // changeType is in the format of "XY" where "X" is the status of the file in the index and "Y" is the status of // the file in the working tree. Some example statuses: // - 'D' == deletion