From 1ddfea35ee7b8305fe50ee841ff89fbd71707cae Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Sat, 18 Dec 2021 12:27:11 -0500 Subject: [PATCH 1/7] Allocate correct capacity for workspace json globs --- cli/internal/context/context.go | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index 028293bee1346..9ce863a437fbf 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -219,15 +219,9 @@ func WithGraph(rootpath string, config *config.Config) Option { // until all parsing is complete // and populate the graph parseJSONWaitGroup := new(errgroup.Group) - justJsons := make([]string, len(spaces)) - for i, space := range spaces { - justJsons[i] = path.Join(space, "package.json") - } - ignore := []string{ - "**/node_modules/**/*", - "**/bower_components/**/*", - "**/test/**/*", - "**/tests/**/*", + justJsons := make([]string, 0, len(spaces)) + for _, space := range spaces { + justJsons = append(justJsons, path.Join(space, "package.json")) } f := globby.GlobFiles(rootpath, justJsons, ignore) From 9aa193050055bba5bdb55a6a6e88b05c158fde45 Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Sat, 18 Dec 2021 12:29:25 -0500 Subject: [PATCH 2/7] Extract workspace ignores into constant slice --- cli/internal/context/context.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index 9ce863a437fbf..45d9036bfba3f 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -224,7 +224,7 @@ func WithGraph(rootpath string, config *config.Config) Option { justJsons = append(justJsons, path.Join(space, "package.json")) } - f := globby.GlobFiles(rootpath, justJsons, ignore) + f := globby.GlobFiles(rootpath, justJsons, getWorkspaceIgnores()) for i, val := range f { _, val := i, val // https://golang.org/doc/faq#closures_and_goroutines @@ -507,3 +507,12 @@ func safeCompileIgnoreFile(filepath string) (*gitignore.GitIgnore, error) { // no op return gitignore.CompileIgnoreLines([]string{}...), nil } + +func getWorkspaceIgnores() []string { + return []string{ + "**/node_modules/**/*", + "**/bower_components/**/*", + "**/test/**/*", + "**/tests/**/*", + } +} From 8b06380040754d5eb894748d1bb0702ca337bf12 Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Sat, 18 Dec 2021 12:36:06 -0500 Subject: [PATCH 3/7] Better encapsulate ColorCache --- cli/internal/context/color_cache.go | 54 +++++++++++++++++++++++++++++ cli/internal/context/context.go | 33 +----------------- cli/internal/run/run.go | 4 +-- 3 files changed, 57 insertions(+), 34 deletions(-) create mode 100644 cli/internal/context/color_cache.go diff --git a/cli/internal/context/color_cache.go b/cli/internal/context/color_cache.go new file mode 100644 index 0000000000000..e11af384c7022 --- /dev/null +++ b/cli/internal/context/color_cache.go @@ -0,0 +1,54 @@ +package context + +import ( + "sync" + + "github.com/fatih/color" +) + +type colorFn = func(format string, a ...interface{}) string + +func getTerminalPackageColors() []colorFn { + return []colorFn{color.CyanString, color.MagentaString, color.GreenString, color.YellowString, color.BlueString} +} + +type ColorCache struct { + mu sync.Mutex + index int + TermColors []colorFn + Cache map[interface{}]colorFn +} + +func NewColorCache() *ColorCache { + return &ColorCache{ + TermColors: getTerminalPackageColors(), + index: 0, + Cache: make(map[interface{}]colorFn), + } +} + +// PrefixColor returns a color function for a given package name +func (c *ColorCache) PrefixColor(name string) colorFn { + c.mu.Lock() + defer c.mu.Unlock() + colorFn, ok := c.Cache[name] + if ok { + return colorFn + } + c.index++ + colorFn = c.TermColors[positiveMod(c.index, 5)] // 5 possible colors + c.Cache[name] = colorFn + return colorFn +} + +// postitiveMod returns a modulo operator like JavaScripts +func positiveMod(x, d int) int { + x = x % d + if x >= 0 { + return x + } + if d < 0 { + return x - d + } + return x + d +} diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index 45d9036bfba3f..86ba521cee2d4 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -15,7 +15,6 @@ import ( "turbo/internal/util" mapset "github.com/deckarep/golang-set" - "github.com/fatih/color" "github.com/google/chrometracing" "github.com/pyr-sh/dag" gitignore "github.com/sabhiram/go-gitignore" @@ -35,19 +34,6 @@ const ( Pnpm ) -type colorFn = func(format string, a ...interface{}) string - -var ( - childProcessIndex = 0 - terminalPackageColors = [5]colorFn{color.CyanString, color.MagentaString, color.GreenString, color.YellowString, color.BlueString} -) - -type ColorCache struct { - sync.Mutex - index int - Cache map[interface{}]colorFn -} - // Context of the CLI type Context struct { Args []string @@ -86,20 +72,6 @@ func New(opts ...Option) (*Context, error) { return &m, nil } -// PrefixColor returns a color function for a given package name -func PrefixColor(c *Context, name *string) colorFn { - c.ColorCache.Lock() - defer c.ColorCache.Unlock() - colorFn, ok := c.ColorCache.Cache[name] - if ok { - return colorFn - } - c.ColorCache.index++ - colorFn = terminalPackageColors[util.PositiveMod(c.ColorCache.index, len(terminalPackageColors))] - c.ColorCache.Cache[name] = colorFn - return colorFn -} - // WithDir specifies the directory where turbo is initiated func WithDir(d string) Option { return func(m *Context) error { @@ -141,10 +113,7 @@ func WithTracer(filename string) Option { func WithGraph(rootpath string, config *config.Config) Option { return func(c *Context) error { c.PackageInfos = make(map[interface{}]*fs.PackageJSON) - c.ColorCache = &ColorCache{ - Cache: make(map[interface{}]colorFn), - index: 0, - } + c.ColorCache = NewColorCache() c.RootNode = ROOT_NODE_NAME c.PendingTaskNodes = make(dag.Set) // Need to ALWAYS have a root node, might as well do it now diff --git a/cli/internal/run/run.go b/cli/internal/run/run.go index 3b886897a90a8..7ed07781ad9ed 100644 --- a/cli/internal/run/run.go +++ b/cli/internal/run/run.go @@ -391,8 +391,8 @@ func (c *RunCommand) Run(args []string) int { tracer := runState.Run(context.GetTaskId(pack.Name, task)) // Create a logger - pref := context.PrefixColor(ctx, &pack.Name) - actualPrefix := pref("%v:%v: ", pack.Name, task) + pref := ctx.ColorCache.PrefixColor(pack.Name) + actualPrefix := pref("%s:%s: ", pack.Name, task) targetUi := &cli.PrefixedUi{ Ui: c.Ui, OutputPrefix: actualPrefix, From 9cc7b6a692cc08008cd7daf77251fe81fa9d328c Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Sat, 18 Dec 2021 12:51:18 -0500 Subject: [PATCH 4/7] Set ExternalDeps capacity --- cli/internal/context/context.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index 86ba521cee2d4..cae8aa87f04b5 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -300,9 +300,9 @@ func (c *Context) ResolveWorkspaceRootDeps() (*fs.PackageJSON, error) { pkg.SubLockfile = make(fs.YarnLockfile) c.ResolveDepGraph(&lockfileWg, pkg.UnresolvedExternalDeps, depSet, seen, pkg) lockfileWg.Wait() - pkg.ExternalDeps = make([]string, depSet.Cardinality()) - for i, v := range depSet.ToSlice() { - pkg.ExternalDeps[i] = v.(string) + pkg.ExternalDeps = make([]string, 0, depSet.Cardinality()) + for _, v := range depSet.ToSlice() { + pkg.ExternalDeps = append(pkg.ExternalDeps, fmt.Sprintf("%s", v)) } sort.Strings(pkg.ExternalDeps) hashOfExternalDeps, err := fs.HashObject(pkg.ExternalDeps) From ad6a497b4c6deaa96ffcee2e183145da1f9e4f0f Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Sat, 18 Dec 2021 12:53:29 -0500 Subject: [PATCH 5/7] Remove redudant word --- cli/internal/context/context.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index cae8aa87f04b5..7718de6baee1c 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -423,7 +423,7 @@ func (c *Context) parsePackageJSON(buildFilePath string) error { if fs.FileExists(buildFilePath) { pkg, err := fs.ReadPackageJSON(buildFilePath) if err != nil { - return fmt.Errorf("error parsing %v: %w", buildFilePath, err) + return fmt.Errorf("parsing %s: %w", buildFilePath, err) } // log.Printf("[TRACE] adding %+v to graph", pkg.Name) From 39650fa7374ffae40af2835ca9557771130d7e81 Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Sat, 18 Dec 2021 13:11:21 -0500 Subject: [PATCH 6/7] Better slice allocation in dependency analysis --- cli/internal/context/context.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index 7718de6baee1c..2bcb673616339 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -297,7 +297,7 @@ func (c *Context) ResolveWorkspaceRootDeps() (*fs.PackageJSON, error) { pkg.UnresolvedExternalDeps[dep] = version } if c.Backend.Name == "nodejs-yarn" && !fs.CheckIfWindows() { - pkg.SubLockfile = make(fs.YarnLockfile) + pkg.SubLockfile = fs.YarnLockfile{} c.ResolveDepGraph(&lockfileWg, pkg.UnresolvedExternalDeps, depSet, seen, pkg) lockfileWg.Wait() pkg.ExternalDeps = make([]string, 0, depSet.Cardinality()) @@ -397,13 +397,13 @@ func (c *Context) populateTopologicGraphForPackageJson(pkg *fs.PackageJSON) erro if internalDepsSet.Len() == 0 { c.TopologicalGraph.Connect(dag.BasicEdge(pkg.Name, ROOT_NODE_NAME)) } - pkg.ExternalDeps = make([]string, externalDepSet.Cardinality()) - for i, v := range externalDepSet.ToSlice() { - pkg.ExternalDeps[i] = v.(string) + pkg.ExternalDeps = make([]string, 0, externalDepSet.Cardinality()) + for _, v := range externalDepSet.ToSlice() { + pkg.ExternalDeps = append(pkg.ExternalDeps, fmt.Sprintf("%s", v)) } - pkg.InternalDeps = make([]string, internalDepsSet.Len()) - for i, v := range internalDepsSet.List() { - pkg.InternalDeps[i] = v.(string) + pkg.InternalDeps = make([]string, 0, internalDepsSet.Len()) + for _, v := range internalDepsSet.List() { + pkg.ExternalDeps = append(pkg.InternalDeps, fmt.Sprintf("%s", v)) } sort.Strings(pkg.InternalDeps) sort.Strings(pkg.ExternalDeps) From 52545a08c51a2299e6b29c44ae956a6736423184 Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Sat, 18 Dec 2021 14:20:37 -0500 Subject: [PATCH 7/7] Fix dep assignment --- cli/internal/context/context.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index 98bcea757d74e..8bd32fe655923 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -297,12 +297,12 @@ func (c *Context) ResolveWorkspaceRootDeps() (*fs.PackageJSON, error) { pkg.UnresolvedExternalDeps[dep] = version } if c.Backend.Name == "nodejs-yarn" && !fs.CheckIfWindows() { - pkg.SubLockfile = fs.YarnLockfile{} + pkg.SubLockfile = make(fs.YarnLockfile) c.ResolveDepGraph(&lockfileWg, pkg.UnresolvedExternalDeps, depSet, seen, pkg) lockfileWg.Wait() pkg.ExternalDeps = make([]string, 0, depSet.Cardinality()) - for _, v := range depSet.ToSlice() { - pkg.ExternalDeps = append(pkg.ExternalDeps, fmt.Sprintf("%s", v)) + for v := range depSet.ToSlice() { + pkg.ExternalDeps = append(pkg.ExternalDeps, fmt.Sprintf("%v", v)) } sort.Strings(pkg.ExternalDeps) hashOfExternalDeps, err := fs.HashObject(pkg.ExternalDeps) @@ -402,12 +402,12 @@ func (c *Context) populateTopologicGraphForPackageJson(pkg *fs.PackageJSON) erro c.TopologicalGraph.Connect(dag.BasicEdge(pkg.Name, ROOT_NODE_NAME)) } pkg.ExternalDeps = make([]string, 0, externalDepSet.Cardinality()) - for _, v := range externalDepSet.ToSlice() { - pkg.ExternalDeps = append(pkg.ExternalDeps, fmt.Sprintf("%s", v)) + for v := range externalDepSet.ToSlice() { + pkg.ExternalDeps = append(pkg.ExternalDeps, fmt.Sprintf("%v", v)) } pkg.InternalDeps = make([]string, 0, internalDepsSet.Len()) - for _, v := range internalDepsSet.List() { - pkg.ExternalDeps = append(pkg.InternalDeps, fmt.Sprintf("%s", v)) + for v := range internalDepsSet.List() { + pkg.ExternalDeps = append(pkg.InternalDeps, fmt.Sprintf("%v", v)) } sort.Strings(pkg.InternalDeps) sort.Strings(pkg.ExternalDeps)