这是indexloc提供的服务,不要输入任何密码
Skip to content

Continue evolving Context towards a data structure that can be sent over the wire #788

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Mar 1, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
203 changes: 87 additions & 116 deletions cli/internal/context/context.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"strings"
"sync"

"github.com/hashicorp/go-hclog"
"github.com/vercel/turborepo/cli/internal/api"
"github.com/vercel/turborepo/cli/internal/backends"
"github.com/vercel/turborepo/cli/internal/config"
Expand All @@ -18,7 +19,6 @@ import (
"github.com/vercel/turborepo/cli/internal/util"

mapset "github.com/deckarep/golang-set"
"github.com/google/chrometracing"
"github.com/pyr-sh/dag"
gitignore "github.com/sabhiram/go-gitignore"
"golang.org/x/sync/errgroup"
Expand All @@ -28,23 +28,15 @@ const GLOBAL_CACHE_KEY = "snozzberries"

// Context of the CLI
type Context struct {
Args []string
PackageInfos map[interface{}]*fs.PackageJSON
PackageNames []string
TopologicalGraph dag.AcyclicGraph
TaskGraph dag.AcyclicGraph
Dir string
RootNode string
RootPackageJSON *fs.PackageJSON
TurboConfig *fs.TurboConfigJSON
GlobalHashableEnvPairs []string
GlobalHashableEnvNames []string
GlobalHash string
TraceFilePath string
Lockfile *fs.YarnLockfile
SCC [][]dag.Vertex
Targets []string
Backend *api.LanguageBackend
PackageInfos map[interface{}]*fs.PackageJSON
PackageNames []string
TopologicalGraph dag.AcyclicGraph
RootNode string
TurboConfig *fs.TurboConfigJSON
GlobalHash string
Lockfile *fs.YarnLockfile
SCC [][]dag.Vertex
Backend *api.LanguageBackend
// Used to arbitrate access to the graph. We parallelise most build operations
// and Go maps aren't natively threadsafe so this is needed.
mutex sync.Mutex
Expand All @@ -65,66 +57,43 @@ func New(opts ...Option) (*Context, error) {
return &m, nil
}

// WithArgs sets the arguments to the command that are used for parsing.
// Remaining arguments can be accessed using your flag set and asking for Args.
// Example: c.Flags().Args().
func WithArgs(args []string) Option {
return func(c *Context) error {
c.Args = args
return nil
}
}

func WithTracer(filename string) Option {
return func(c *Context) error {
if filename != "" {
chrometracing.EnableTracing()
c.TraceFilePath = filename
}
return nil
}
}

func WithGraph(rootpath string, config *config.Config) Option {
return func(c *Context) error {
c.PackageInfos = make(map[interface{}]*fs.PackageJSON)
c.RootNode = core.ROOT_NODE_NAME
// Need to ALWAYS have a root node, might as well do it now
c.TaskGraph.Add(core.ROOT_NODE_NAME)

packageJSONPath := filepath.Join(rootpath, "package.json")
pkg, err := fs.ReadPackageJSON(packageJSONPath)
rootPackageJSON, err := fs.ReadPackageJSON(packageJSONPath)
if err != nil {
return fmt.Errorf("package.json: %w", err)
}
c.RootPackageJSON = pkg

// If turbo.json exists, we use that
// If pkg.Turbo exists, we warn about running the migration
// Use pkg.Turbo if turbo.json doesn't exist
// If neither exists, it's a fatal error
turboJSONPath := filepath.Join(rootpath, "turbo.json")
if !fs.FileExists(turboJSONPath) {
if pkg.LegacyTurboConfig == nil {
if rootPackageJSON.LegacyTurboConfig == nil {
// TODO: suggestion on how to create one
return fmt.Errorf("Could not find turbo.json. Follow directions at https://turborepo.org/docs/getting-started to create one")
} else {
log.Println("[WARNING] Turbo configuration now lives in \"turbo.json\". Migrate to turbo.json by running \"npx @turbo/codemod create-turbo-config\"")
c.TurboConfig = pkg.LegacyTurboConfig
c.TurboConfig = rootPackageJSON.LegacyTurboConfig
}
} else {
turbo, err := fs.ReadTurboConfigJSON(turboJSONPath)
if err != nil {
return fmt.Errorf("turbo.json: %w", err)
}
c.TurboConfig = turbo
if pkg.LegacyTurboConfig != nil {
if rootPackageJSON.LegacyTurboConfig != nil {
log.Println("[WARNING] Ignoring legacy \"turbo\" key in package.json, using turbo.json instead. Consider deleting the \"turbo\" key from package.json")
pkg.LegacyTurboConfig = nil
rootPackageJSON.LegacyTurboConfig = nil
}
}

if backend, err := backends.GetBackend(rootpath, pkg); err != nil {
if backend, err := backends.GetBackend(rootpath, rootPackageJSON); err != nil {
return err
} else {
c.Backend = backend
Expand All @@ -139,7 +108,7 @@ func WithGraph(rootpath string, config *config.Config) Option {
c.Lockfile = lockfile
}

if c.ResolveWorkspaceRootDeps() != nil {
if c.ResolveWorkspaceRootDeps(rootPackageJSON) != nil {
return err
}

Expand All @@ -149,71 +118,8 @@ func WithGraph(rootpath string, config *config.Config) Option {
return fmt.Errorf("could not detect workspaces: %w", err)
}

// Calculate the global hash
globalDeps := make(util.Set)

// Calculate global file and env var dependencies
if len(c.TurboConfig.GlobalDependencies) > 0 {
var globs []string
for _, v := range c.TurboConfig.GlobalDependencies {
if strings.HasPrefix(v, "$") {
trimmed := strings.TrimPrefix(v, "$")
c.GlobalHashableEnvNames = append(c.GlobalHashableEnvNames, trimmed)
c.GlobalHashableEnvPairs = append(c.GlobalHashableEnvPairs, fmt.Sprintf("%v=%v", trimmed, os.Getenv(trimmed)))
} else {
globs = append(globs, v)
}
}

if len(globs) > 0 {
f := globby.GlobFiles(rootpath, globs, []string{})
for _, val := range f {
globalDeps.Add(val)
}
}
}

// get system env vars for hashing purposes, these include any variable that includes "TURBO"
// that is NOT TURBO_TOKEN or TURBO_TEAM or TURBO_BINARY_PATH.
names, pairs := getHashableTurboEnvVarsFromOs()
c.GlobalHashableEnvNames = append(c.GlobalHashableEnvNames, names...)
c.GlobalHashableEnvPairs = append(c.GlobalHashableEnvPairs, pairs...)
// sort them for consistent hashing
sort.Strings(c.GlobalHashableEnvNames)
sort.Strings(c.GlobalHashableEnvPairs)
config.Logger.Debug("global hash env vars", "vars", c.GlobalHashableEnvNames)

if !util.IsYarn(c.Backend.Name) {
// If we are not in Yarn, add the specfile and lockfile to global deps
globalDeps.Add(c.Backend.Specfile)
globalDeps.Add(c.Backend.Lockfile)
}

globalFileHashMap, err := fs.GitHashForFiles(globalDeps.UnsafeListOfStrings(), rootpath)
if err != nil {
return fmt.Errorf("error hashing files. make sure that git has been initialized %w", err)
}
globalHashable := struct {
globalFileHashMap map[string]string
rootExternalDepsHash string
hashedSortedEnvPairs []string
globalCacheKey string
}{
globalFileHashMap: globalFileHashMap,
rootExternalDepsHash: pkg.ExternalDepsHash,
hashedSortedEnvPairs: c.GlobalHashableEnvPairs,
globalCacheKey: GLOBAL_CACHE_KEY,
}
globalHash, err := fs.HashObject(globalHashable)
if err != nil {
return fmt.Errorf("error hashing global dependencies %w", err)
}
globalHash, err := calculateGlobalHash(rootpath, rootPackageJSON, c.TurboConfig.GlobalDependencies, c.Backend, config.Logger, os.Environ())
c.GlobalHash = globalHash
targets, err := GetTargetsFromArguments(c.Args, c.TurboConfig)
if err != nil {
return err
}
c.Targets = targets
// We will parse all package.json's simultaneously. We use a
// waitgroup because we cannot fully populate the graph (the next step)
// until all parsing is complete
Expand Down Expand Up @@ -308,10 +214,10 @@ func (c *Context) loadPackageDepsHash(pkg *fs.PackageJSON) error {
return nil
}

func (c *Context) ResolveWorkspaceRootDeps() error {
func (c *Context) ResolveWorkspaceRootDeps(rootPackageJSON *fs.PackageJSON) error {
seen := mapset.NewSet()
var lockfileWg sync.WaitGroup
pkg := c.RootPackageJSON
pkg := rootPackageJSON
depSet := mapset.NewSet()
pkg.UnresolvedExternalDeps = make(map[string]string)
for dep, version := range pkg.Dependencies {
Expand Down Expand Up @@ -537,10 +443,10 @@ func getWorkspaceIgnores() []string {

// getHashableTurboEnvVarsFromOs returns a list of environment variables names and
// that are safe to include in the global hash
func getHashableTurboEnvVarsFromOs() ([]string, []string) {
func getHashableTurboEnvVarsFromOs(env []string) ([]string, []string) {
var justNames []string
var pairs []string
for _, e := range os.Environ() {
for _, e := range env {
kv := strings.SplitN(e, "=", 2)
if strings.Contains(kv[0], "THASH") {
justNames = append(justNames, kv[0])
Expand All @@ -550,3 +456,68 @@ func getHashableTurboEnvVarsFromOs() ([]string, []string) {

return justNames, pairs
}

func calculateGlobalHash(rootpath string, rootPackageJSON *fs.PackageJSON, externalGlobalDependencies []string, backend *api.LanguageBackend, logger hclog.Logger, env []string) (string, error) {
// Calculate the global hash
globalDeps := make(util.Set)

globalHashableEnvNames := []string{}
globalHashableEnvPairs := []string{}
// Calculate global file and env var dependencies
if len(externalGlobalDependencies) > 0 {
var globs []string
for _, v := range externalGlobalDependencies {
if strings.HasPrefix(v, "$") {
trimmed := strings.TrimPrefix(v, "$")
globalHashableEnvNames = append(globalHashableEnvNames, trimmed)
globalHashableEnvPairs = append(globalHashableEnvPairs, fmt.Sprintf("%v=%v", trimmed, os.Getenv(trimmed)))
} else {
globs = append(globs, v)
}
}

if len(globs) > 0 {
f := globby.GlobFiles(rootpath, globs, []string{})
for _, val := range f {
globalDeps.Add(val)
}
}
}

// get system env vars for hashing purposes, these include any variable that includes "TURBO"
// that is NOT TURBO_TOKEN or TURBO_TEAM or TURBO_BINARY_PATH.
names, pairs := getHashableTurboEnvVarsFromOs(env)
globalHashableEnvNames = append(globalHashableEnvNames, names...)
globalHashableEnvPairs = append(globalHashableEnvPairs, pairs...)
// sort them for consistent hashing
sort.Strings(globalHashableEnvNames)
sort.Strings(globalHashableEnvPairs)
logger.Debug("global hash env vars", "vars", globalHashableEnvNames)

if !util.IsYarn(backend.Name) {
// If we are not in Yarn, add the specfile and lockfile to global deps
globalDeps.Add(backend.Specfile)
globalDeps.Add(backend.Lockfile)
}

globalFileHashMap, err := fs.GitHashForFiles(globalDeps.UnsafeListOfStrings(), rootpath)
if err != nil {
return "", fmt.Errorf("error hashing files. make sure that git has been initialized %w", err)
}
globalHashable := struct {
globalFileHashMap map[string]string
rootExternalDepsHash string
hashedSortedEnvPairs []string
globalCacheKey string
}{
globalFileHashMap: globalFileHashMap,
rootExternalDepsHash: rootPackageJSON.ExternalDepsHash,
hashedSortedEnvPairs: globalHashableEnvPairs,
globalCacheKey: GLOBAL_CACHE_KEY,
}
globalHash, err := fs.HashObject(globalHashable)
if err != nil {
return "", fmt.Errorf("error hashing global dependencies %w", err)
}
return globalHash, nil
}
19 changes: 10 additions & 9 deletions cli/internal/context/context_test.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
package context

import (
"os"
"reflect"
"testing"

"github.com/vercel/turborepo/cli/internal/fs"
)

Expand Down Expand Up @@ -96,14 +96,15 @@ func TestGetTargetsFromArguments(t *testing.T) {
}

func Test_getHashableTurboEnvVarsFromOs(t *testing.T) {
os.Setenv("SOME_ENV_VAR", "excluded")
os.Setenv("SOME_OTHER_ENV_VAR", "excluded")
os.Setenv("FIRST_THASH_ENV_VAR", "first")
os.Setenv("TURBO_TOKEN", "never")
os.Setenv("SOME_OTHER_THASH_ENV_VAR", "second")
os.Setenv("TURBO_TEAM", "never")

gotNames, gotPairs := getHashableTurboEnvVarsFromOs()
env := []string{
"SOME_ENV_VAR=excluded",
"SOME_OTHER_ENV_VAR=excluded",
"FIRST_THASH_ENV_VAR=first",
"TURBO_TOKEN=never",
"SOME_OTHER_THASH_ENV_VAR=second",
"TURBO_TEAM=never",
}
gotNames, gotPairs := getHashableTurboEnvVarsFromOs(env)
wantNames := []string{"FIRST_THASH_ENV_VAR", "SOME_OTHER_THASH_ENV_VAR"}
wantPairs := []string{"FIRST_THASH_ENV_VAR=first", "SOME_OTHER_THASH_ENV_VAR=second"}
if !reflect.DeepEqual(wantNames, gotNames) {
Expand Down
2 changes: 1 addition & 1 deletion cli/internal/prune/prune.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ func (c *PruneCommand) Run(args []string) int {
c.logError(c.Config.Logger, "", err)
return 1
}
ctx, err := context.New(context.WithTracer(""), context.WithArgs(args), context.WithGraph(pruneOptions.cwd, c.Config))
ctx, err := context.New(context.WithGraph(pruneOptions.cwd, c.Config))

if err != nil {
c.logError(c.Config.Logger, "", fmt.Errorf("could not construct graph: %w", err))
Expand Down
9 changes: 7 additions & 2 deletions cli/internal/run/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,11 +143,16 @@ func (c *RunCommand) Run(args []string) int {

c.Config.Cache.Dir = runOptions.cacheFolder

ctx, err := context.New(context.WithTracer(runOptions.profile), context.WithArgs(args), context.WithGraph(runOptions.cwd, c.Config))
ctx, err := context.New(context.WithGraph(runOptions.cwd, c.Config))
if err != nil {
c.logError(c.Config.Logger, "", err)
return 1
}
targets, err := context.GetTargetsFromArguments(args, ctx.TurboConfig)
if err != nil {
c.logError(c.Config.Logger, "", fmt.Errorf("failed to resolve targets: %w", err))
return 1
}

gitRepoRoot, err := fs.FindupFrom(".git", runOptions.cwd)
if err != nil {
Expand Down Expand Up @@ -308,7 +313,7 @@ func (c *RunCommand) Run(args []string) int {
RootNode: ctx.RootNode,
}
rs := &runSpec{
Targets: ctx.Targets,
Targets: targets,
FilteredPkgs: filteredPkgs,
Opts: runOptions,
}
Expand Down