From 450ee72493a096c2808cc9dfe9c1497631e2e7d0 Mon Sep 17 00:00:00 2001 From: Greg Soltis Date: Wed, 16 Feb 2022 14:22:21 -0800 Subject: [PATCH 1/4] Remove unnecessary TaskGraph from context --- cli/internal/context/context.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index c5eba238263de..7a83b4f709189 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -32,7 +32,6 @@ type Context struct { PackageInfos map[interface{}]*fs.PackageJSON PackageNames []string TopologicalGraph dag.AcyclicGraph - TaskGraph dag.AcyclicGraph Dir string RootNode string RootPackageJSON *fs.PackageJSON @@ -89,8 +88,6 @@ func WithGraph(rootpath string, config *config.Config) Option { return func(c *Context) error { c.PackageInfos = make(map[interface{}]*fs.PackageJSON) c.RootNode = core.ROOT_NODE_NAME - // Need to ALWAYS have a root node, might as well do it now - c.TaskGraph.Add(core.ROOT_NODE_NAME) packageJSONPath := filepath.Join(rootpath, "package.json") pkg, err := fs.ReadPackageJSON(packageJSONPath) From 73a8ee015c4215e6098028385cf025c4d26fd02d Mon Sep 17 00:00:00 2001 From: Greg Soltis Date: Wed, 16 Feb 2022 21:09:59 -0800 Subject: [PATCH 2/4] Extract global hash --- cli/internal/context/context.go | 160 ++++++++++++++------------- cli/internal/context/context_test.go | 19 ++-- 2 files changed, 93 insertions(+), 86 deletions(-) diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index 7a83b4f709189..a6e348c3a99d1 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -9,6 +9,7 @@ import ( "strings" "sync" + "github.com/hashicorp/go-hclog" "github.com/vercel/turborepo/cli/internal/api" "github.com/vercel/turborepo/cli/internal/backends" "github.com/vercel/turborepo/cli/internal/config" @@ -28,22 +29,20 @@ const GLOBAL_CACHE_KEY = "snozzberries" // Context of the CLI type Context struct { - Args []string - PackageInfos map[interface{}]*fs.PackageJSON - PackageNames []string - TopologicalGraph dag.AcyclicGraph - Dir string - RootNode string - RootPackageJSON *fs.PackageJSON - TurboConfig *fs.TurboConfigJSON - GlobalHashableEnvPairs []string - GlobalHashableEnvNames []string - GlobalHash string - TraceFilePath string - Lockfile *fs.YarnLockfile - SCC [][]dag.Vertex - Targets []string - Backend *api.LanguageBackend + Args []string + PackageInfos map[interface{}]*fs.PackageJSON + PackageNames []string + TopologicalGraph dag.AcyclicGraph + Dir string + RootNode string + RootPackageJSON *fs.PackageJSON + TurboConfig *fs.TurboConfigJSON + GlobalHash string + TraceFilePath string + Lockfile *fs.YarnLockfile + SCC [][]dag.Vertex + Targets []string + Backend *api.LanguageBackend // Used to arbitrate access to the graph. We parallelise most build operations // and Go maps aren't natively threadsafe so this is needed. mutex sync.Mutex @@ -146,65 +145,7 @@ func WithGraph(rootpath string, config *config.Config) Option { return fmt.Errorf("could not detect workspaces: %w", err) } - // Calculate the global hash - globalDeps := make(util.Set) - - // Calculate global file and env var dependencies - if len(c.TurboConfig.GlobalDependencies) > 0 { - var globs []string - for _, v := range c.TurboConfig.GlobalDependencies { - if strings.HasPrefix(v, "$") { - trimmed := strings.TrimPrefix(v, "$") - c.GlobalHashableEnvNames = append(c.GlobalHashableEnvNames, trimmed) - c.GlobalHashableEnvPairs = append(c.GlobalHashableEnvPairs, fmt.Sprintf("%v=%v", trimmed, os.Getenv(trimmed))) - } else { - globs = append(globs, v) - } - } - - if len(globs) > 0 { - f := globby.GlobFiles(rootpath, globs, []string{}) - for _, val := range f { - globalDeps.Add(val) - } - } - } - - // get system env vars for hashing purposes, these include any variable that includes "TURBO" - // that is NOT TURBO_TOKEN or TURBO_TEAM or TURBO_BINARY_PATH. - names, pairs := getHashableTurboEnvVarsFromOs() - c.GlobalHashableEnvNames = append(c.GlobalHashableEnvNames, names...) - c.GlobalHashableEnvPairs = append(c.GlobalHashableEnvPairs, pairs...) - // sort them for consistent hashing - sort.Strings(c.GlobalHashableEnvNames) - sort.Strings(c.GlobalHashableEnvPairs) - config.Logger.Debug("global hash env vars", "vars", c.GlobalHashableEnvNames) - - if !util.IsYarn(c.Backend.Name) { - // If we are not in Yarn, add the specfile and lockfile to global deps - globalDeps.Add(c.Backend.Specfile) - globalDeps.Add(c.Backend.Lockfile) - } - - globalFileHashMap, err := fs.GitHashForFiles(globalDeps.UnsafeListOfStrings(), rootpath) - if err != nil { - return fmt.Errorf("error hashing files. make sure that git has been initialized %w", err) - } - globalHashable := struct { - globalFileHashMap map[string]string - rootExternalDepsHash string - hashedSortedEnvPairs []string - globalCacheKey string - }{ - globalFileHashMap: globalFileHashMap, - rootExternalDepsHash: pkg.ExternalDepsHash, - hashedSortedEnvPairs: c.GlobalHashableEnvPairs, - globalCacheKey: GLOBAL_CACHE_KEY, - } - globalHash, err := fs.HashObject(globalHashable) - if err != nil { - return fmt.Errorf("error hashing global dependencies %w", err) - } + globalHash, err := calculateGlobalHash(rootpath, pkg, c.TurboConfig.GlobalDependencies, c.Backend, config.Logger, os.Environ()) c.GlobalHash = globalHash targets, err := GetTargetsFromArguments(c.Args, c.TurboConfig) if err != nil { @@ -534,10 +475,10 @@ func getWorkspaceIgnores() []string { // getHashableTurboEnvVarsFromOs returns a list of environment variables names and // that are safe to include in the global hash -func getHashableTurboEnvVarsFromOs() ([]string, []string) { +func getHashableTurboEnvVarsFromOs(env []string) ([]string, []string) { var justNames []string var pairs []string - for _, e := range os.Environ() { + for _, e := range env { kv := strings.SplitN(e, "=", 2) if strings.Contains(kv[0], "THASH") { justNames = append(justNames, kv[0]) @@ -547,3 +488,68 @@ func getHashableTurboEnvVarsFromOs() ([]string, []string) { return justNames, pairs } + +func calculateGlobalHash(rootpath string, rootPackageJSON *fs.PackageJSON, externalGlobalDependencies []string, backend *api.LanguageBackend, logger hclog.Logger, env []string) (string, error) { + // Calculate the global hash + globalDeps := make(util.Set) + + globalHashableEnvNames := []string{} + globalHashableEnvPairs := []string{} + // Calculate global file and env var dependencies + if len(externalGlobalDependencies) > 0 { + var globs []string + for _, v := range externalGlobalDependencies { + if strings.HasPrefix(v, "$") { + trimmed := strings.TrimPrefix(v, "$") + globalHashableEnvNames = append(globalHashableEnvNames, trimmed) + globalHashableEnvPairs = append(globalHashableEnvPairs, fmt.Sprintf("%v=%v", trimmed, os.Getenv(trimmed))) + } else { + globs = append(globs, v) + } + } + + if len(globs) > 0 { + f := globby.GlobFiles(rootpath, globs, []string{}) + for _, val := range f { + globalDeps.Add(val) + } + } + } + + // get system env vars for hashing purposes, these include any variable that includes "TURBO" + // that is NOT TURBO_TOKEN or TURBO_TEAM or TURBO_BINARY_PATH. + names, pairs := getHashableTurboEnvVarsFromOs(env) + globalHashableEnvNames = append(globalHashableEnvNames, names...) + globalHashableEnvPairs = append(globalHashableEnvPairs, pairs...) + // sort them for consistent hashing + sort.Strings(globalHashableEnvNames) + sort.Strings(globalHashableEnvPairs) + logger.Debug("global hash env vars", "vars", globalHashableEnvNames) + + if !util.IsYarn(backend.Name) { + // If we are not in Yarn, add the specfile and lockfile to global deps + globalDeps.Add(backend.Specfile) + globalDeps.Add(backend.Lockfile) + } + + globalFileHashMap, err := fs.GitHashForFiles(globalDeps.UnsafeListOfStrings(), rootpath) + if err != nil { + return "", fmt.Errorf("error hashing files. make sure that git has been initialized %w", err) + } + globalHashable := struct { + globalFileHashMap map[string]string + rootExternalDepsHash string + hashedSortedEnvPairs []string + globalCacheKey string + }{ + globalFileHashMap: globalFileHashMap, + rootExternalDepsHash: rootPackageJSON.ExternalDepsHash, + hashedSortedEnvPairs: globalHashableEnvPairs, + globalCacheKey: GLOBAL_CACHE_KEY, + } + globalHash, err := fs.HashObject(globalHashable) + if err != nil { + return "", fmt.Errorf("error hashing global dependencies %w", err) + } + return globalHash, nil +} diff --git a/cli/internal/context/context_test.go b/cli/internal/context/context_test.go index 498e6f24aa0f5..315361b8a53c3 100644 --- a/cli/internal/context/context_test.go +++ b/cli/internal/context/context_test.go @@ -1,9 +1,9 @@ package context import ( - "os" "reflect" "testing" + "github.com/vercel/turborepo/cli/internal/fs" ) @@ -96,14 +96,15 @@ func TestGetTargetsFromArguments(t *testing.T) { } func Test_getHashableTurboEnvVarsFromOs(t *testing.T) { - os.Setenv("SOME_ENV_VAR", "excluded") - os.Setenv("SOME_OTHER_ENV_VAR", "excluded") - os.Setenv("FIRST_THASH_ENV_VAR", "first") - os.Setenv("TURBO_TOKEN", "never") - os.Setenv("SOME_OTHER_THASH_ENV_VAR", "second") - os.Setenv("TURBO_TEAM", "never") - - gotNames, gotPairs := getHashableTurboEnvVarsFromOs() + env := []string{ + "SOME_ENV_VAR=excluded", + "SOME_OTHER_ENV_VAR=excluded", + "FIRST_THASH_ENV_VAR=first", + "TURBO_TOKEN=never", + "SOME_OTHER_THASH_ENV_VAR=second", + "TURBO_TEAM=never", + } + gotNames, gotPairs := getHashableTurboEnvVarsFromOs(env) wantNames := []string{"FIRST_THASH_ENV_VAR", "SOME_OTHER_THASH_ENV_VAR"} wantPairs := []string{"FIRST_THASH_ENV_VAR=first", "SOME_OTHER_THASH_ENV_VAR=second"} if !reflect.DeepEqual(wantNames, gotNames) { From 040f8fdbdbf547517b53a0972452f13fade0281f Mon Sep 17 00:00:00 2001 From: Greg Soltis Date: Fri, 25 Feb 2022 17:35:36 -0800 Subject: [PATCH 3/4] Remove some fields from Context that are either unused or do not need to be part of that struct --- cli/internal/context/context.go | 30 ------------------------------ cli/internal/prune/prune.go | 2 +- cli/internal/run/run.go | 9 +++++++-- 3 files changed, 8 insertions(+), 33 deletions(-) diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index a6e348c3a99d1..315f3d96fc218 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -19,7 +19,6 @@ import ( "github.com/vercel/turborepo/cli/internal/util" mapset "github.com/deckarep/golang-set" - "github.com/google/chrometracing" "github.com/pyr-sh/dag" gitignore "github.com/sabhiram/go-gitignore" "golang.org/x/sync/errgroup" @@ -29,19 +28,15 @@ const GLOBAL_CACHE_KEY = "snozzberries" // Context of the CLI type Context struct { - Args []string PackageInfos map[interface{}]*fs.PackageJSON PackageNames []string TopologicalGraph dag.AcyclicGraph - Dir string RootNode string RootPackageJSON *fs.PackageJSON TurboConfig *fs.TurboConfigJSON GlobalHash string - TraceFilePath string Lockfile *fs.YarnLockfile SCC [][]dag.Vertex - Targets []string Backend *api.LanguageBackend // Used to arbitrate access to the graph. We parallelise most build operations // and Go maps aren't natively threadsafe so this is needed. @@ -63,26 +58,6 @@ func New(opts ...Option) (*Context, error) { return &m, nil } -// WithArgs sets the arguments to the command that are used for parsing. -// Remaining arguments can be accessed using your flag set and asking for Args. -// Example: c.Flags().Args(). -func WithArgs(args []string) Option { - return func(c *Context) error { - c.Args = args - return nil - } -} - -func WithTracer(filename string) Option { - return func(c *Context) error { - if filename != "" { - chrometracing.EnableTracing() - c.TraceFilePath = filename - } - return nil - } -} - func WithGraph(rootpath string, config *config.Config) Option { return func(c *Context) error { c.PackageInfos = make(map[interface{}]*fs.PackageJSON) @@ -147,11 +122,6 @@ func WithGraph(rootpath string, config *config.Config) Option { globalHash, err := calculateGlobalHash(rootpath, pkg, c.TurboConfig.GlobalDependencies, c.Backend, config.Logger, os.Environ()) c.GlobalHash = globalHash - targets, err := GetTargetsFromArguments(c.Args, c.TurboConfig) - if err != nil { - return err - } - c.Targets = targets // We will parse all package.json's simultaneously. We use a // waitgroup because we cannot fully populate the graph (the next step) // until all parsing is complete diff --git a/cli/internal/prune/prune.go b/cli/internal/prune/prune.go index 48f2b0b8486e5..b1e79b8cc6e07 100644 --- a/cli/internal/prune/prune.go +++ b/cli/internal/prune/prune.go @@ -100,7 +100,7 @@ func (c *PruneCommand) Run(args []string) int { c.logError(c.Config.Logger, "", err) return 1 } - ctx, err := context.New(context.WithTracer(""), context.WithArgs(args), context.WithGraph(pruneOptions.cwd, c.Config)) + ctx, err := context.New(context.WithGraph(pruneOptions.cwd, c.Config)) if err != nil { c.logError(c.Config.Logger, "", fmt.Errorf("could not construct graph: %w", err)) diff --git a/cli/internal/run/run.go b/cli/internal/run/run.go index 6e1d5967d102e..0e42876b2c52d 100644 --- a/cli/internal/run/run.go +++ b/cli/internal/run/run.go @@ -143,11 +143,16 @@ func (c *RunCommand) Run(args []string) int { c.Config.Cache.Dir = runOptions.cacheFolder - ctx, err := context.New(context.WithTracer(runOptions.profile), context.WithArgs(args), context.WithGraph(runOptions.cwd, c.Config)) + ctx, err := context.New(context.WithGraph(runOptions.cwd, c.Config)) if err != nil { c.logError(c.Config.Logger, "", err) return 1 } + targets, err := context.GetTargetsFromArguments(args, ctx.TurboConfig) + if err != nil { + c.logError(c.Config.Logger, "", fmt.Errorf("failed to resolve targets: %w", err)) + return 1 + } gitRepoRoot, err := fs.FindupFrom(".git", runOptions.cwd) if err != nil { @@ -308,7 +313,7 @@ func (c *RunCommand) Run(args []string) int { RootNode: ctx.RootNode, } rs := &runSpec{ - Targets: ctx.Targets, + Targets: targets, FilteredPkgs: filteredPkgs, Opts: runOptions, } From 655498e901791d495ec11f203f3bd689ed863ea1 Mon Sep 17 00:00:00 2001 From: Greg Soltis Date: Sun, 27 Feb 2022 15:03:34 -0800 Subject: [PATCH 4/4] Drop root package JSON from context --- cli/internal/context/context.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index 315f3d96fc218..026f801719eac 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -32,7 +32,6 @@ type Context struct { PackageNames []string TopologicalGraph dag.AcyclicGraph RootNode string - RootPackageJSON *fs.PackageJSON TurboConfig *fs.TurboConfigJSON GlobalHash string Lockfile *fs.YarnLockfile @@ -64,11 +63,10 @@ func WithGraph(rootpath string, config *config.Config) Option { c.RootNode = core.ROOT_NODE_NAME packageJSONPath := filepath.Join(rootpath, "package.json") - pkg, err := fs.ReadPackageJSON(packageJSONPath) + rootPackageJSON, err := fs.ReadPackageJSON(packageJSONPath) if err != nil { return fmt.Errorf("package.json: %w", err) } - c.RootPackageJSON = pkg // If turbo.json exists, we use that // If pkg.Turbo exists, we warn about running the migration @@ -76,12 +74,12 @@ func WithGraph(rootpath string, config *config.Config) Option { // If neither exists, it's a fatal error turboJSONPath := filepath.Join(rootpath, "turbo.json") if !fs.FileExists(turboJSONPath) { - if pkg.LegacyTurboConfig == nil { + if rootPackageJSON.LegacyTurboConfig == nil { // TODO: suggestion on how to create one return fmt.Errorf("Could not find turbo.json. Follow directions at https://turborepo.org/docs/getting-started to create one") } else { log.Println("[WARNING] Turbo configuration now lives in \"turbo.json\". Migrate to turbo.json by running \"npx @turbo/codemod create-turbo-config\"") - c.TurboConfig = pkg.LegacyTurboConfig + c.TurboConfig = rootPackageJSON.LegacyTurboConfig } } else { turbo, err := fs.ReadTurboConfigJSON(turboJSONPath) @@ -89,13 +87,13 @@ func WithGraph(rootpath string, config *config.Config) Option { return fmt.Errorf("turbo.json: %w", err) } c.TurboConfig = turbo - if pkg.LegacyTurboConfig != nil { + if rootPackageJSON.LegacyTurboConfig != nil { log.Println("[WARNING] Ignoring legacy \"turbo\" key in package.json, using turbo.json instead. Consider deleting the \"turbo\" key from package.json") - pkg.LegacyTurboConfig = nil + rootPackageJSON.LegacyTurboConfig = nil } } - if backend, err := backends.GetBackend(rootpath, pkg); err != nil { + if backend, err := backends.GetBackend(rootpath, rootPackageJSON); err != nil { return err } else { c.Backend = backend @@ -110,7 +108,7 @@ func WithGraph(rootpath string, config *config.Config) Option { c.Lockfile = lockfile } - if c.ResolveWorkspaceRootDeps() != nil { + if c.ResolveWorkspaceRootDeps(rootPackageJSON) != nil { return err } @@ -120,7 +118,7 @@ func WithGraph(rootpath string, config *config.Config) Option { return fmt.Errorf("could not detect workspaces: %w", err) } - globalHash, err := calculateGlobalHash(rootpath, pkg, c.TurboConfig.GlobalDependencies, c.Backend, config.Logger, os.Environ()) + globalHash, err := calculateGlobalHash(rootpath, rootPackageJSON, c.TurboConfig.GlobalDependencies, c.Backend, config.Logger, os.Environ()) c.GlobalHash = globalHash // We will parse all package.json's simultaneously. We use a // waitgroup because we cannot fully populate the graph (the next step) @@ -216,10 +214,10 @@ func (c *Context) loadPackageDepsHash(pkg *fs.PackageJSON) error { return nil } -func (c *Context) ResolveWorkspaceRootDeps() error { +func (c *Context) ResolveWorkspaceRootDeps(rootPackageJSON *fs.PackageJSON) error { seen := mapset.NewSet() var lockfileWg sync.WaitGroup - pkg := c.RootPackageJSON + pkg := rootPackageJSON depSet := mapset.NewSet() pkg.UnresolvedExternalDeps = make(map[string]string) for dep, version := range pkg.Dependencies {