这是indexloc提供的服务,不要输入任何密码
Skip to content

Add new string filter abstraction and more e2e tests #421

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Dec 23, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ cli/npm/turbo-sunos-64/bin
cli/npm/turbo-windows-32/turbo.exe
cli/npm/turbo-windows-64/turbo.exe
cli/npm/turbo-windows-arm64/turbo.exe
cli/scripts/turbo-*
!/npm/turbo-windows-32/bin
!/npm/turbo-windows-64/bin
!/npm/turbo-install/bin
Expand Down
127 changes: 61 additions & 66 deletions cli/internal/run/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,11 @@ import (
"turbo/internal/ui"
"turbo/internal/util"
"turbo/internal/util/browser"
"turbo/internal/util/filter"

"github.com/pyr-sh/dag"

"github.com/fatih/color"
glob "github.com/gobwas/glob"
"github.com/hashicorp/go-hclog"
"github.com/mitchellh/cli"
"github.com/pkg/errors"
Expand Down Expand Up @@ -139,14 +139,14 @@ func (c *RunCommand) Run(args []string) int {
c.logWarning(c.Config.Logger, "", err)
}

ignoreGlobs, err := convertStringsToGlobs(runOptions.ignore)
ignoreGlob, err := filter.Compile(runOptions.ignore)
if err != nil {
c.logError(c.Config.Logger, "", fmt.Errorf("invalid ignore globs: %w", err))
return 1
}
globalDeps, err := convertStringsToGlobs(runOptions.globalDeps)
globalDepsGlob, err := filter.Compile(runOptions.globalDeps)
if err != nil {
c.logError(c.Config.Logger, "", fmt.Errorf("invalid global deps: %w", err))
c.logError(c.Config.Logger, "", fmt.Errorf("invalid global deps glob: %w", err))
return 1
}
hasRepoGlobalFileChanged := false
Expand All @@ -156,23 +156,23 @@ func (c *RunCommand) Run(args []string) int {
}

ignoreSet := make(util.Set)

for _, f := range changedFiles {
for _, g := range globalDeps {
if g.Match(f) {
if globalDepsGlob != nil {
for _, f := range changedFiles {
if globalDepsGlob.Match(f) {
hasRepoGlobalFileChanged = true
break
}
}
}

for _, f := range changedFiles {
for _, g := range ignoreGlobs {
if g.Match(f) {
if ignoreGlob != nil {
for _, f := range changedFiles {
if ignoreGlob.Match(f) {
ignoreSet.Add(f)
}
}
}

filteredChangedFiles := make(util.Set)
// Ignore any changed files in the ignore set
for _, c := range changedFiles {
Expand All @@ -197,7 +197,6 @@ func (c *RunCommand) Run(args []string) int {
// Scoped packages
// Unwind scope globs
scopePkgs, err := getScopedPackages(ctx, runOptions.scope)

if err != nil {
c.logError(c.Config.Logger, "", fmt.Errorf("Invalid scope: %w", err))
return 1
Expand Down Expand Up @@ -229,37 +228,40 @@ func (c *RunCommand) Run(args []string) int {
c.Ui.Output(fmt.Sprintf(ui.Dim("• Packages changed since %s: %s"), runOptions.since, strings.Join(filteredPkgs.UnsafeListOfStrings(), ", ")))
} else if scopePkgs.Len() > 0 {
filteredPkgs = scopePkgs
c.Ui.Output(fmt.Sprintf(ui.Dim("• Packages in scope: %v"), strings.Join(scopePkgs.UnsafeListOfStrings(), ", ")))
} else {
for _, f := range ctx.PackageNames {
filteredPkgs.Add(f)
}
}

if runOptions.deps {
if runOptions.includeDependents {
// perf??? this is duplicative from the step above
for _, changed := range filteredPkgs {
descenders, err := ctx.TopologicalGraph.Descendents(changed)
for _, pkg := range filteredPkgs {
descenders, err := ctx.TopologicalGraph.Descendents(pkg)
if err != nil {
c.logError(c.Config.Logger, "", fmt.Errorf("error calculating affected packages: %w", err))
return 1
}
// filteredPkgs.Add(changed)
c.Config.Logger.Debug("dependents", "pkg", pkg, "value", descenders.List())
for _, d := range descenders {
filteredPkgs.Add(d)
// we need to exlcude the fake root node
// since it is not a real package
if d != ctx.RootNode {
filteredPkgs.Add(d)
}
}
}
c.Config.Logger.Debug("running with dependents")
}

if runOptions.ancestors {
for _, changed := range filteredPkgs {
ancestors, err := ctx.TopologicalGraph.Ancestors(changed)
if runOptions.includeDependencies {
for _, pkg := range filteredPkgs {
ancestors, err := ctx.TopologicalGraph.Ancestors(pkg)
if err != nil {
log.Printf("error getting dependency %v", err)
return 1
}
c.Config.Logger.Debug("dependencies", ancestors)
c.Config.Logger.Debug("dependencies", "pkg", pkg, "value", ancestors.List())
for _, d := range ancestors {
// we need to exlcude the fake root node
// since it is not a real package
Expand All @@ -270,9 +272,10 @@ func (c *RunCommand) Run(args []string) int {
}
c.Config.Logger.Debug(ui.Dim("running with dependencies"))
}
c.Config.Logger.Debug("execution scope", "packages", strings.Join(filteredPkgs.UnsafeListOfStrings(), ", "))
c.Config.Logger.Debug("global hash", "value", ctx.GlobalHash)

packagesInScope := filteredPkgs.UnsafeListOfStrings()
sort.Strings(packagesInScope)
c.Ui.Output(fmt.Sprintf(ui.Dim("• Packages in scope: %v"), strings.Join(packagesInScope, ", ")))
c.Config.Logger.Debug("local cache folder", "path", runOptions.cacheFolder)
fs.EnsureDir(runOptions.cacheFolder)
turboCache := cache.New(c.Config)
Expand Down Expand Up @@ -682,9 +685,9 @@ func (c *RunCommand) Run(args []string) int {

type RunOptions struct {
// Whether to include dependent impacted consumers in execution (defaults to true)
deps bool
// Whether to include ancestors (pkg.dependencies) in execution (defaults to false)
ancestors bool
includeDependents bool
// Whether to include includeDependencies (pkg.dependencies) in execution (defaults to false)
includeDependencies bool
// List of globs of file paths to ignore from exection scope calculation
ignore []string
// Whether to stream log outputs
Expand Down Expand Up @@ -720,17 +723,17 @@ type RunOptions struct {

func getDefaultRunOptions() *RunOptions {
return &RunOptions{
bail: true,
deps: true,
parallel: false,
concurrency: 10,
dotGraph: "",
ancestors: false,
cache: true,
profile: "", // empty string does no tracing
forceExecution: false,
stream: true,
only: false,
bail: true,
includeDependents: true,
parallel: false,
concurrency: 10,
dotGraph: "",
includeDependencies: false,
cache: true,
profile: "", // empty string does no tracing
forceExecution: false,
stream: true,
only: false,
}
}

Expand All @@ -750,38 +753,38 @@ func parseRunArgs(args []string, cwd string) (*RunOptions, error) {
} else if strings.HasPrefix(arg, "--") {
switch {
case strings.HasPrefix(arg, "--since="):
if len(arg[len("--since="):]) > 1 {
if len(arg[len("--since="):]) > 0 {
runOptions.since = arg[len("--since="):]
}
case strings.HasPrefix(arg, "--scope="):
if len(arg[len("--scope="):]) > 1 {
if len(arg[len("--scope="):]) > 0 {
runOptions.scope = append(runOptions.scope, arg[len("--scope="):])
}
case strings.HasPrefix(arg, "--ignore="):
if len(arg[len("--ignore="):]) > 1 {
if len(arg[len("--ignore="):]) > 0 {
runOptions.ignore = append(runOptions.ignore, arg[len("--ignore="):])
}
case strings.HasPrefix(arg, "--global-deps="):
if len(arg[len("--global-deps="):]) > 1 {
if len(arg[len("--global-deps="):]) > 0 {
runOptions.globalDeps = append(runOptions.globalDeps, arg[len("--global-deps="):])
}
case strings.HasPrefix(arg, "--cwd="):
if len(arg[len("--cwd="):]) > 1 {
if len(arg[len("--cwd="):]) > 0 {
runOptions.cwd = arg[len("--cwd="):]
} else {
runOptions.cwd = cwd
}
case strings.HasPrefix(arg, "--parallel"):
runOptions.parallel = true
case strings.HasPrefix(arg, "--profile="): // this one must com before the next
if len(arg[len("--profile="):]) > 1 {
if len(arg[len("--profile="):]) > 0 {
runOptions.profile = arg[len("--profile="):]
}
case strings.HasPrefix(arg, "--profile"):
runOptions.profile = fmt.Sprintf("%v-profile.json", time.Now().UnixNano())

case strings.HasPrefix(arg, "--no-deps"):
runOptions.deps = false
runOptions.includeDependents = false
case strings.HasPrefix(arg, "--no-cache"):
runOptions.cache = true
case strings.HasPrefix(arg, "--cacheFolder"):
Expand All @@ -797,7 +800,7 @@ func parseRunArgs(args []string, cwd string) (*RunOptions, error) {
runOptions.stream = true

case strings.HasPrefix(arg, "--graph="): // this one must com before the next
if len(arg[len("--graph="):]) > 1 {
if len(arg[len("--graph="):]) > 0 {
runOptions.dotGraph = arg[len("--graph="):]
}
case strings.HasPrefix(arg, "--graph"):
Expand All @@ -816,8 +819,10 @@ func parseRunArgs(args []string, cwd string) (*RunOptions, error) {
}
}
case strings.HasPrefix(arg, "--includeDependencies"):
log.Printf("[WARNING] The --includeDependencies flag has renamed to --include-dependencies for consistency. Please use `--include-dependencies` instead")
runOptions.includeDependencies = true
case strings.HasPrefix(arg, "--include-dependencies"):
runOptions.ancestors = true
runOptions.includeDependencies = true
case strings.HasPrefix(arg, "--only"):
runOptions.only = true
case strings.HasPrefix(arg, "--team"):
Expand Down Expand Up @@ -845,32 +850,22 @@ func parseRunArgs(args []string, cwd string) (*RunOptions, error) {
return runOptions, nil
}

// convertStringsToGlobs converts string glob patterns to an array glob.Glob instances.
func convertStringsToGlobs(patterns []string) (globss []glob.Glob, err error) {
var globs = make([]glob.Glob, 0, len(patterns))
for _, pattern := range patterns {
g, err := glob.Compile(pattern)
if err != nil {
return nil, err
}
globs = append(globs, g)
}

return globs, nil
}

// getScopedPackages returns a set of package names in scope for a given list of glob patterns
func getScopedPackages(ctx *context.Context, scopePatterns []string) (scopePkgs util.Set, err error) {
scopeGlobs, err := convertStringsToGlobs(scopePatterns)
if err != nil {
return nil, fmt.Errorf("invalid glob pattern %w", err)
}
var scopedPkgs = make(util.Set)
if len(scopePatterns) == 0 {
return scopePkgs, nil
}
glob, err := filter.Compile(scopePatterns)
if err != nil {
return nil, err
}
for _, f := range ctx.PackageNames {
for _, g := range scopeGlobs {
if g.Match(f) {
scopedPkgs.Add(f)
}
if glob.Match(f) {
scopedPkgs.Add(f)
}
}

Expand Down
Loading