diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index c5eba238263de..026f801719eac 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -9,6 +9,7 @@ import ( "strings" "sync" + "github.com/hashicorp/go-hclog" "github.com/vercel/turborepo/cli/internal/api" "github.com/vercel/turborepo/cli/internal/backends" "github.com/vercel/turborepo/cli/internal/config" @@ -18,7 +19,6 @@ import ( "github.com/vercel/turborepo/cli/internal/util" mapset "github.com/deckarep/golang-set" - "github.com/google/chrometracing" "github.com/pyr-sh/dag" gitignore "github.com/sabhiram/go-gitignore" "golang.org/x/sync/errgroup" @@ -28,23 +28,15 @@ const GLOBAL_CACHE_KEY = "snozzberries" // Context of the CLI type Context struct { - Args []string - PackageInfos map[interface{}]*fs.PackageJSON - PackageNames []string - TopologicalGraph dag.AcyclicGraph - TaskGraph dag.AcyclicGraph - Dir string - RootNode string - RootPackageJSON *fs.PackageJSON - TurboConfig *fs.TurboConfigJSON - GlobalHashableEnvPairs []string - GlobalHashableEnvNames []string - GlobalHash string - TraceFilePath string - Lockfile *fs.YarnLockfile - SCC [][]dag.Vertex - Targets []string - Backend *api.LanguageBackend + PackageInfos map[interface{}]*fs.PackageJSON + PackageNames []string + TopologicalGraph dag.AcyclicGraph + RootNode string + TurboConfig *fs.TurboConfigJSON + GlobalHash string + Lockfile *fs.YarnLockfile + SCC [][]dag.Vertex + Backend *api.LanguageBackend // Used to arbitrate access to the graph. We parallelise most build operations // and Go maps aren't natively threadsafe so this is needed. mutex sync.Mutex @@ -65,39 +57,16 @@ func New(opts ...Option) (*Context, error) { return &m, nil } -// WithArgs sets the arguments to the command that are used for parsing. -// Remaining arguments can be accessed using your flag set and asking for Args. -// Example: c.Flags().Args(). -func WithArgs(args []string) Option { - return func(c *Context) error { - c.Args = args - return nil - } -} - -func WithTracer(filename string) Option { - return func(c *Context) error { - if filename != "" { - chrometracing.EnableTracing() - c.TraceFilePath = filename - } - return nil - } -} - func WithGraph(rootpath string, config *config.Config) Option { return func(c *Context) error { c.PackageInfos = make(map[interface{}]*fs.PackageJSON) c.RootNode = core.ROOT_NODE_NAME - // Need to ALWAYS have a root node, might as well do it now - c.TaskGraph.Add(core.ROOT_NODE_NAME) packageJSONPath := filepath.Join(rootpath, "package.json") - pkg, err := fs.ReadPackageJSON(packageJSONPath) + rootPackageJSON, err := fs.ReadPackageJSON(packageJSONPath) if err != nil { return fmt.Errorf("package.json: %w", err) } - c.RootPackageJSON = pkg // If turbo.json exists, we use that // If pkg.Turbo exists, we warn about running the migration @@ -105,12 +74,12 @@ func WithGraph(rootpath string, config *config.Config) Option { // If neither exists, it's a fatal error turboJSONPath := filepath.Join(rootpath, "turbo.json") if !fs.FileExists(turboJSONPath) { - if pkg.LegacyTurboConfig == nil { + if rootPackageJSON.LegacyTurboConfig == nil { // TODO: suggestion on how to create one return fmt.Errorf("Could not find turbo.json. Follow directions at https://turborepo.org/docs/getting-started to create one") } else { log.Println("[WARNING] Turbo configuration now lives in \"turbo.json\". Migrate to turbo.json by running \"npx @turbo/codemod create-turbo-config\"") - c.TurboConfig = pkg.LegacyTurboConfig + c.TurboConfig = rootPackageJSON.LegacyTurboConfig } } else { turbo, err := fs.ReadTurboConfigJSON(turboJSONPath) @@ -118,13 +87,13 @@ func WithGraph(rootpath string, config *config.Config) Option { return fmt.Errorf("turbo.json: %w", err) } c.TurboConfig = turbo - if pkg.LegacyTurboConfig != nil { + if rootPackageJSON.LegacyTurboConfig != nil { log.Println("[WARNING] Ignoring legacy \"turbo\" key in package.json, using turbo.json instead. Consider deleting the \"turbo\" key from package.json") - pkg.LegacyTurboConfig = nil + rootPackageJSON.LegacyTurboConfig = nil } } - if backend, err := backends.GetBackend(rootpath, pkg); err != nil { + if backend, err := backends.GetBackend(rootpath, rootPackageJSON); err != nil { return err } else { c.Backend = backend @@ -139,7 +108,7 @@ func WithGraph(rootpath string, config *config.Config) Option { c.Lockfile = lockfile } - if c.ResolveWorkspaceRootDeps() != nil { + if c.ResolveWorkspaceRootDeps(rootPackageJSON) != nil { return err } @@ -149,71 +118,8 @@ func WithGraph(rootpath string, config *config.Config) Option { return fmt.Errorf("could not detect workspaces: %w", err) } - // Calculate the global hash - globalDeps := make(util.Set) - - // Calculate global file and env var dependencies - if len(c.TurboConfig.GlobalDependencies) > 0 { - var globs []string - for _, v := range c.TurboConfig.GlobalDependencies { - if strings.HasPrefix(v, "$") { - trimmed := strings.TrimPrefix(v, "$") - c.GlobalHashableEnvNames = append(c.GlobalHashableEnvNames, trimmed) - c.GlobalHashableEnvPairs = append(c.GlobalHashableEnvPairs, fmt.Sprintf("%v=%v", trimmed, os.Getenv(trimmed))) - } else { - globs = append(globs, v) - } - } - - if len(globs) > 0 { - f := globby.GlobFiles(rootpath, globs, []string{}) - for _, val := range f { - globalDeps.Add(val) - } - } - } - - // get system env vars for hashing purposes, these include any variable that includes "TURBO" - // that is NOT TURBO_TOKEN or TURBO_TEAM or TURBO_BINARY_PATH. - names, pairs := getHashableTurboEnvVarsFromOs() - c.GlobalHashableEnvNames = append(c.GlobalHashableEnvNames, names...) - c.GlobalHashableEnvPairs = append(c.GlobalHashableEnvPairs, pairs...) - // sort them for consistent hashing - sort.Strings(c.GlobalHashableEnvNames) - sort.Strings(c.GlobalHashableEnvPairs) - config.Logger.Debug("global hash env vars", "vars", c.GlobalHashableEnvNames) - - if !util.IsYarn(c.Backend.Name) { - // If we are not in Yarn, add the specfile and lockfile to global deps - globalDeps.Add(c.Backend.Specfile) - globalDeps.Add(c.Backend.Lockfile) - } - - globalFileHashMap, err := fs.GitHashForFiles(globalDeps.UnsafeListOfStrings(), rootpath) - if err != nil { - return fmt.Errorf("error hashing files. make sure that git has been initialized %w", err) - } - globalHashable := struct { - globalFileHashMap map[string]string - rootExternalDepsHash string - hashedSortedEnvPairs []string - globalCacheKey string - }{ - globalFileHashMap: globalFileHashMap, - rootExternalDepsHash: pkg.ExternalDepsHash, - hashedSortedEnvPairs: c.GlobalHashableEnvPairs, - globalCacheKey: GLOBAL_CACHE_KEY, - } - globalHash, err := fs.HashObject(globalHashable) - if err != nil { - return fmt.Errorf("error hashing global dependencies %w", err) - } + globalHash, err := calculateGlobalHash(rootpath, rootPackageJSON, c.TurboConfig.GlobalDependencies, c.Backend, config.Logger, os.Environ()) c.GlobalHash = globalHash - targets, err := GetTargetsFromArguments(c.Args, c.TurboConfig) - if err != nil { - return err - } - c.Targets = targets // We will parse all package.json's simultaneously. We use a // waitgroup because we cannot fully populate the graph (the next step) // until all parsing is complete @@ -308,10 +214,10 @@ func (c *Context) loadPackageDepsHash(pkg *fs.PackageJSON) error { return nil } -func (c *Context) ResolveWorkspaceRootDeps() error { +func (c *Context) ResolveWorkspaceRootDeps(rootPackageJSON *fs.PackageJSON) error { seen := mapset.NewSet() var lockfileWg sync.WaitGroup - pkg := c.RootPackageJSON + pkg := rootPackageJSON depSet := mapset.NewSet() pkg.UnresolvedExternalDeps = make(map[string]string) for dep, version := range pkg.Dependencies { @@ -537,10 +443,10 @@ func getWorkspaceIgnores() []string { // getHashableTurboEnvVarsFromOs returns a list of environment variables names and // that are safe to include in the global hash -func getHashableTurboEnvVarsFromOs() ([]string, []string) { +func getHashableTurboEnvVarsFromOs(env []string) ([]string, []string) { var justNames []string var pairs []string - for _, e := range os.Environ() { + for _, e := range env { kv := strings.SplitN(e, "=", 2) if strings.Contains(kv[0], "THASH") { justNames = append(justNames, kv[0]) @@ -550,3 +456,68 @@ func getHashableTurboEnvVarsFromOs() ([]string, []string) { return justNames, pairs } + +func calculateGlobalHash(rootpath string, rootPackageJSON *fs.PackageJSON, externalGlobalDependencies []string, backend *api.LanguageBackend, logger hclog.Logger, env []string) (string, error) { + // Calculate the global hash + globalDeps := make(util.Set) + + globalHashableEnvNames := []string{} + globalHashableEnvPairs := []string{} + // Calculate global file and env var dependencies + if len(externalGlobalDependencies) > 0 { + var globs []string + for _, v := range externalGlobalDependencies { + if strings.HasPrefix(v, "$") { + trimmed := strings.TrimPrefix(v, "$") + globalHashableEnvNames = append(globalHashableEnvNames, trimmed) + globalHashableEnvPairs = append(globalHashableEnvPairs, fmt.Sprintf("%v=%v", trimmed, os.Getenv(trimmed))) + } else { + globs = append(globs, v) + } + } + + if len(globs) > 0 { + f := globby.GlobFiles(rootpath, globs, []string{}) + for _, val := range f { + globalDeps.Add(val) + } + } + } + + // get system env vars for hashing purposes, these include any variable that includes "TURBO" + // that is NOT TURBO_TOKEN or TURBO_TEAM or TURBO_BINARY_PATH. + names, pairs := getHashableTurboEnvVarsFromOs(env) + globalHashableEnvNames = append(globalHashableEnvNames, names...) + globalHashableEnvPairs = append(globalHashableEnvPairs, pairs...) + // sort them for consistent hashing + sort.Strings(globalHashableEnvNames) + sort.Strings(globalHashableEnvPairs) + logger.Debug("global hash env vars", "vars", globalHashableEnvNames) + + if !util.IsYarn(backend.Name) { + // If we are not in Yarn, add the specfile and lockfile to global deps + globalDeps.Add(backend.Specfile) + globalDeps.Add(backend.Lockfile) + } + + globalFileHashMap, err := fs.GitHashForFiles(globalDeps.UnsafeListOfStrings(), rootpath) + if err != nil { + return "", fmt.Errorf("error hashing files. make sure that git has been initialized %w", err) + } + globalHashable := struct { + globalFileHashMap map[string]string + rootExternalDepsHash string + hashedSortedEnvPairs []string + globalCacheKey string + }{ + globalFileHashMap: globalFileHashMap, + rootExternalDepsHash: rootPackageJSON.ExternalDepsHash, + hashedSortedEnvPairs: globalHashableEnvPairs, + globalCacheKey: GLOBAL_CACHE_KEY, + } + globalHash, err := fs.HashObject(globalHashable) + if err != nil { + return "", fmt.Errorf("error hashing global dependencies %w", err) + } + return globalHash, nil +} diff --git a/cli/internal/context/context_test.go b/cli/internal/context/context_test.go index 498e6f24aa0f5..315361b8a53c3 100644 --- a/cli/internal/context/context_test.go +++ b/cli/internal/context/context_test.go @@ -1,9 +1,9 @@ package context import ( - "os" "reflect" "testing" + "github.com/vercel/turborepo/cli/internal/fs" ) @@ -96,14 +96,15 @@ func TestGetTargetsFromArguments(t *testing.T) { } func Test_getHashableTurboEnvVarsFromOs(t *testing.T) { - os.Setenv("SOME_ENV_VAR", "excluded") - os.Setenv("SOME_OTHER_ENV_VAR", "excluded") - os.Setenv("FIRST_THASH_ENV_VAR", "first") - os.Setenv("TURBO_TOKEN", "never") - os.Setenv("SOME_OTHER_THASH_ENV_VAR", "second") - os.Setenv("TURBO_TEAM", "never") - - gotNames, gotPairs := getHashableTurboEnvVarsFromOs() + env := []string{ + "SOME_ENV_VAR=excluded", + "SOME_OTHER_ENV_VAR=excluded", + "FIRST_THASH_ENV_VAR=first", + "TURBO_TOKEN=never", + "SOME_OTHER_THASH_ENV_VAR=second", + "TURBO_TEAM=never", + } + gotNames, gotPairs := getHashableTurboEnvVarsFromOs(env) wantNames := []string{"FIRST_THASH_ENV_VAR", "SOME_OTHER_THASH_ENV_VAR"} wantPairs := []string{"FIRST_THASH_ENV_VAR=first", "SOME_OTHER_THASH_ENV_VAR=second"} if !reflect.DeepEqual(wantNames, gotNames) { diff --git a/cli/internal/prune/prune.go b/cli/internal/prune/prune.go index 48f2b0b8486e5..b1e79b8cc6e07 100644 --- a/cli/internal/prune/prune.go +++ b/cli/internal/prune/prune.go @@ -100,7 +100,7 @@ func (c *PruneCommand) Run(args []string) int { c.logError(c.Config.Logger, "", err) return 1 } - ctx, err := context.New(context.WithTracer(""), context.WithArgs(args), context.WithGraph(pruneOptions.cwd, c.Config)) + ctx, err := context.New(context.WithGraph(pruneOptions.cwd, c.Config)) if err != nil { c.logError(c.Config.Logger, "", fmt.Errorf("could not construct graph: %w", err)) diff --git a/cli/internal/run/run.go b/cli/internal/run/run.go index 6e1d5967d102e..0e42876b2c52d 100644 --- a/cli/internal/run/run.go +++ b/cli/internal/run/run.go @@ -143,11 +143,16 @@ func (c *RunCommand) Run(args []string) int { c.Config.Cache.Dir = runOptions.cacheFolder - ctx, err := context.New(context.WithTracer(runOptions.profile), context.WithArgs(args), context.WithGraph(runOptions.cwd, c.Config)) + ctx, err := context.New(context.WithGraph(runOptions.cwd, c.Config)) if err != nil { c.logError(c.Config.Logger, "", err) return 1 } + targets, err := context.GetTargetsFromArguments(args, ctx.TurboConfig) + if err != nil { + c.logError(c.Config.Logger, "", fmt.Errorf("failed to resolve targets: %w", err)) + return 1 + } gitRepoRoot, err := fs.FindupFrom(".git", runOptions.cwd) if err != nil { @@ -308,7 +313,7 @@ func (c *RunCommand) Run(args []string) int { RootNode: ctx.RootNode, } rs := &runSpec{ - Targets: ctx.Targets, + Targets: targets, FilteredPkgs: filteredPkgs, Opts: runOptions, }