From 612c5398c1e977ad22f534b4e2d2f4c3e3f5521a Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Wed, 15 Dec 2021 20:59:55 -0500 Subject: [PATCH 1/6] Fix doublestar workspace bug --- cli/internal/context/context.go | 40 ++-- cli/internal/fs/globby/globby.go | 179 ++++++++++++++ cli/internal/fs/globby/globby_test.go | 323 ++++++++++++++++++++++++++ 3 files changed, 526 insertions(+), 16 deletions(-) create mode 100644 cli/internal/fs/globby/globby.go create mode 100644 cli/internal/fs/globby/globby_test.go diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index c2efd16bd4f0b..aa3bd32037c40 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -2,7 +2,7 @@ package context import ( "fmt" - "log" + "path" "path/filepath" "sort" "strings" @@ -11,6 +11,7 @@ import ( "turbo/internal/backends" "turbo/internal/config" "turbo/internal/fs" + "turbo/internal/fs/globby" "turbo/internal/util" "github.com/bmatcuk/doublestar" @@ -181,7 +182,7 @@ func WithGraph(rootpath string, config *config.Config) Option { if len(pkg.Turbo.GlobalDependencies) > 0 { for _, value := range pkg.Turbo.GlobalDependencies { - f, err := filepath.Glob(value) + f, err := doublestar.Glob(value) if err != nil { return fmt.Errorf("error parsing global dependencies glob %v: %w", value, err) } @@ -237,18 +238,26 @@ func WithGraph(rootpath string, config *config.Config) Option { // until all parsing is complete // and populate the graph parseJSONWaitGroup := new(errgroup.Group) - for _, value := range spaces { - f, err := doublestar.Glob(value) - if err != nil { - log.Fatalf("Error parsing workspaces glob %v", value) - } + justJsons := make([]string, len(spaces)) + for i, space := range spaces { + justJsons[i] = path.Join(space, "package.json") + } + f := globby.Match(justJsons, globby.Option{ + BaseDir: rootpath, + CheckDot: true, + Excludes: []string{ + "**/node_modules/**/*", + "**/bower_components/**/*", + "**/test/**/*", + "**/tests/**/*", + }, + }) - for i, val := range f { - _, val := i, val // https://golang.org/doc/faq#closures_and_goroutines - parseJSONWaitGroup.Go(func() error { - return c.parsePackageJSON(val) - }) - } + for i, val := range f { + _, val := i, val // https://golang.org/doc/faq#closures_and_goroutines + parseJSONWaitGroup.Go(func() error { + return c.parsePackageJSON(val) + }) } if err := parseJSONWaitGroup.Wait(); err != nil { @@ -442,10 +451,9 @@ func (c *Context) populateTopologicGraphForPackageJson(pkg *fs.PackageJSON) erro return nil } -func (c *Context) parsePackageJSON(fileName string) error { +func (c *Context) parsePackageJSON(buildFilePath string) error { c.mutex.Lock() defer c.mutex.Unlock() - buildFilePath := filepath.Join(fileName, "package.json") // log.Printf("[TRACE] reading package.json : %+v", buildFilePath) if fs.FileExists(buildFilePath) { @@ -457,7 +465,7 @@ func (c *Context) parsePackageJSON(fileName string) error { // log.Printf("[TRACE] adding %+v to graph", pkg.Name) c.TopologicalGraph.Add(pkg.Name) pkg.PackageJSONPath = buildFilePath - pkg.Dir = fileName + pkg.Dir = filepath.Dir(buildFilePath) c.PackageInfos[pkg.Name] = pkg c.PackageNames = append(c.PackageNames, pkg.Name) } diff --git a/cli/internal/fs/globby/globby.go b/cli/internal/fs/globby/globby.go new file mode 100644 index 0000000000000..e7f14c0550d38 --- /dev/null +++ b/cli/internal/fs/globby/globby.go @@ -0,0 +1,179 @@ +package globby + +import ( + "fmt" + "os" + "path/filepath" + "regexp" +) + +type Option struct { + BaseDir string + CheckDot bool + RelativeReturn bool + Excludes []string +} + +/* + * Glob all patterns + */ +func Match(patterns []string, opt Option) []string { + var allFiles []string + patterns, opt, err := completeOpt(patterns, opt) + if err != nil { + fmt.Printf("Magth err: [%v]\n", err) + return allFiles + } + for _, pattern := range patterns { + files := find(pattern, opt) + if files == nil || len(*files) == 0 { + continue + } + allFiles = append(allFiles, *files...) + } + return allFiles +} + +func find(pattern string, opt Option) *[]string { + // match ./some/path/**/* + if regexTest("\\*\\*", pattern) || + !regexTest("\\*", pattern) { // Dirname + return findRecr(pattern, opt) + } + // match ./some/path/* + if regexTest("\\*", pattern) { + return findDir(pattern, opt) + } + return nil +} + +// find under centain directory +func findDir(pattern string, opt Option) *[]string { + var list []string + files, err := filepath.Glob(pattern) + if err != nil { + fmt.Printf("err: [%v]\n", err) + return &list + } + for _, fullpath := range files { + path, err := filepath.Rel(opt.BaseDir, fullpath) + if err != nil { + continue + } + if checkExclude(opt, path) { + continue + } + if opt.RelativeReturn { + list = append(list, path) + } else { + list = append(list, fullpath) + } + } + return &list +} + +// find recursively +func findRecr(pattern string, opt Option) *[]string { + dir := strReplace(pattern, "\\*\\*.+", "") + afterMacth := "" + matchAfterFlag := false + if regexTest("\\*", pattern) { + afterMacth = strReplace(pattern, ".+\\*", "") + matchAfterFlag = len(afterMacth) > 0 + } + + var list []string + err := filepath.Walk(dir, func(fullpath string, f os.FileInfo, err error) error { + if !opt.CheckDot && regexTest("^\\.", f.Name()) { + if f.IsDir() { + return filepath.SkipDir + } + return nil + } + if f.IsDir() { + return nil + } + path, _ := filepath.Rel(opt.BaseDir, fullpath) + if checkExclude(opt, path) { + return nil + } + if !opt.RelativeReturn { + path = fullpath + } + if !matchAfterFlag { + list = append(list, path) + return nil + } + if regexTest(afterMacth+"$", path) { + list = append(list, path) + } + return nil + }) + if err != nil { + fmt.Printf("err: [%v]\n", err) + } + return &list +} + +// check and complete the options +func completeOpt(srcPatterns []string, opt Option) ([]string, Option, error) { + if len(opt.BaseDir) == 0 { + curDir, err := os.Getwd() + if err != nil { + panic(err) + } + opt.BaseDir = curDir + } + + var patterns []string + for _, pattern := range srcPatterns { + // TODO: check no "tmp/*", use "tmp" or "tmp/*.ext" instead + + if regexTest("^\\!", pattern) { + opt.Excludes = append(opt.Excludes, strReplace(pattern, "^\\!", "")) + continue + } + if regexTest("^\\.", pattern) || // like ./dist + !regexTest("^\\/", pattern) { // like dist + patterns = append(patterns, filepath.Join(opt.BaseDir, pattern)) + continue + } + patterns = append(patterns, pattern) + } + return patterns, opt, nil +} + +// check if path should be excluded +func checkExclude(opt Option, path string) bool { + // if exludes dirs + for _, exclude := range opt.Excludes { + rule := exclude + if regexTest("\\*\\*", exclude) { + rule = strReplace(exclude, "\\*\\*/\\*+?", ".+") + } else if regexTest("\\*", exclude) { + rule = strReplace(exclude, "\\*", "[^/]+") + } + if regexTest("^"+rule, path) { + return true // ignore + } + } + return false +} + +// Check if regex match the "src" string +func regexTest(re string, src string) bool { + matched, err := regexp.MatchString(re, src) + if err != nil { + return false + } + if matched { + return true + } + return false +} + +// "dest" replace "text" pattern with "repl" +func strReplace(dest, text, repl string) string { + re := regexp.MustCompile(text) + return re.ReplaceAllString(dest, repl) +} diff --git a/cli/internal/fs/globby/globby_test.go b/cli/internal/fs/globby/globby_test.go new file mode 100644 index 0000000000000..7be102397d872 --- /dev/null +++ b/cli/internal/fs/globby/globby_test.go @@ -0,0 +1,323 @@ +package globby + +import ( + "os" + "path/filepath" + "reflect" + "sort" + "testing" +) + +/* + * Test ignore .git + */ +func TestIgnoreDotGitFiles(t *testing.T) { + // Init test files + curDir, _ := os.Getwd() + tmpDir := filepath.Join(curDir, "./tmp") + defer os.RemoveAll(tmpDir) + makeTmpFiles(tmpDir, []string{ + ".git/file", + ".gitignore", + "app.js", + }) + + // Match the patterns + files := Match([]string{"."}, Option{BaseDir: tmpDir}) + // Expected match files: + expected := []string{"app.js"} + if checkFiles(tmpDir, files, expected) { + t.Errorf("files not match, expected %v, but got %v", expected, files) + } +} + +/* + * Match "./** /*.jpg" + */ +func TestMathAllImg(t *testing.T) { + // Init test files + curDir, _ := os.Getwd() + tmpDir := filepath.Join(curDir, "./tmp") + defer os.RemoveAll(tmpDir) + makeTmpFiles(tmpDir, []string{ + "app.js", + "src/test.js", + "image/footer.jpg", + "image/logo.jpg", + "image/user/avatar.jpg", + }) + // Match the patterns + files := Match([]string{ + "./**/*.jpg", + }, Option{BaseDir: tmpDir}) + // Expected match files: + expected := []string{ + "image/footer.jpg", + "image/logo.jpg", + "image/user/avatar.jpg", + } + if checkFiles(tmpDir, files, expected) { + t.Errorf("files not match, expected %v, but got %v", expected, files) + } +} + +/* + * Match "src/*.js" + */ +func TestSignleStarFiles(t *testing.T) { + // Init test files + curDir, _ := os.Getwd() + tmpDir := filepath.Join(curDir, "./tmp") + defer os.RemoveAll(tmpDir) + makeTmpFiles(tmpDir, []string{ + ".git", + "app.js", + "package.json", + "src/router.js", + "src/store.js", + "src/api/home.js", + "src/api/user.js", + "src/api/test.js", + }) + + patterns := []string{ + "src/*.js", + } + + // Match the patterns + files := Match(patterns, Option{BaseDir: tmpDir}) + // Expected match files: + expected := []string{ + "src/router.js", + "src/store.js", + } + if checkFiles(tmpDir, files, expected) { + t.Errorf("files not match, expected %v, but got %v", expected, files) + } +} + +/* + * Match "src/api" + */ +func TestDirMatch(t *testing.T) { + // Init test files + curDir, _ := os.Getwd() + tmpDir := filepath.Join(curDir, "./tmp") + defer os.RemoveAll(tmpDir) + makeTmpFiles(tmpDir, []string{ + ".git", + "app.js", + "package.json", + "src/router.js", + "src/store.js", + "src/api/home.js", + "src/api/user.js", + "src/api/test.js", + }) + + patterns := []string{ + "src/api", + } + + // Match the patterns + files := Match(patterns, Option{BaseDir: tmpDir}) + // Expected match files: + expected := []string{ + "src/api/home.js", + "src/api/user.js", + "src/api/test.js", + } + if checkFiles(tmpDir, files, expected) { + t.Errorf("files not match, expected %v, but got %v", expected, files) + } +} + +/* + * Match "/**" + "/*" + */ +func TestDirStar(t *testing.T) { + // Init test files + curDir, _ := os.Getwd() + tmpDir := filepath.Join(curDir, "./tmp") + defer os.RemoveAll(tmpDir) + makeTmpFiles(tmpDir, []string{ + ".git", + "app.js", + "package.json", + "src/router.js", + "src/store.js", + "src/api/home.js", + "src/api/user.js", + "src/api/test.js", + }) + + patterns := []string{ + "src/**/*", + } + + // Match the patterns + files := Match(patterns, Option{BaseDir: tmpDir}) + // Expected match files: + expected := []string{ + "src/router.js", + "src/store.js", + "src/api/home.js", + "src/api/user.js", + "src/api/test.js", + } + if checkFiles(tmpDir, files, expected) { + t.Errorf("files not match, expected %v, but got %v", expected, files) + } +} + +/* + * Match "/**" + "/*.js" + */ +func TestDirStar2(t *testing.T) { + // Init test files + curDir, _ := os.Getwd() + tmpDir := filepath.Join(curDir, "./tmp") + defer os.RemoveAll(tmpDir) + makeTmpFiles(tmpDir, []string{ + ".git", + "app.js", + "package.json", + "src/router.js", + "src/store.js", + "src/store.ts", + "src/api/home.js", + "src/api/user.js", + "src/api/test.js", + }) + + patterns := []string{ + "src/**/*.js", + } + + // Match the patterns + files := Match(patterns, Option{BaseDir: tmpDir}) + // Expected match files: + expected := []string{ + "src/router.js", + "src/store.js", + "src/api/home.js", + "src/api/user.js", + "src/api/test.js", + } + if checkFiles(tmpDir, files, expected) { + t.Errorf("files not match, expected %v, but got %v", expected, files) + } +} + +/* + * Match "/**" + "/*.js" + * ignore files in the match items + */ +func TestDirIgnoreFile(t *testing.T) { + // Init test files + curDir, _ := os.Getwd() + tmpDir := filepath.Join(curDir, "./tmp") + defer os.RemoveAll(tmpDir) + makeTmpFiles(tmpDir, []string{ + ".git", + "app.js", + "package.json", + "src/router.js", + "src/store.js", + "src/store.ts", + "src/api/home.js", + "src/api/user.js", + "src/api/test.js", + "src/service/home.js", + "src/service/user.js", + "src/service/test.js", + }) + + patterns := []string{ + "src/**/*.js", + "!src/service/home.js", + } + + // Match the patterns + files := Match(patterns, Option{BaseDir: tmpDir}) + // Expected match files: + expected := []string{ + "src/router.js", + "src/store.js", + "src/api/home.js", + "src/api/user.js", + "src/api/test.js", + "src/service/user.js", + "src/service/test.js", + } + if checkFiles(tmpDir, files, expected) { + t.Errorf("files not match, expected %v, but got %v", expected, files) + } +} + +/* + * Match "/**" + "/*.js" + * ignore dir in the match items + */ +func TestDirIgnoreDir(t *testing.T) { + // Init test files + curDir, _ := os.Getwd() + tmpDir := filepath.Join(curDir, "./tmp") + defer os.RemoveAll(tmpDir) + makeTmpFiles(tmpDir, []string{ + ".git", + "app.js", + "package.json", + "src/router.js", + "src/store.js", + "src/store.ts", + "src/api/home.js", + "src/api/user.js", + "src/api/test.js", + "src/service/home.js", + "src/service/user.js", + "src/service/test.js", + }) + + patterns := []string{ + "src/**/*.js", + "!src/service", + } + + // Match the patterns + files := Match(patterns, Option{BaseDir: tmpDir}) + // Expected match files: + expected := []string{ + "src/router.js", + "src/store.js", + "src/api/home.js", + "src/api/user.js", + "src/api/test.js", + } + if checkFiles(tmpDir, files, expected) { + t.Errorf("files not match, expected %v, but got %v", expected, files) + } +} + +func TestMain(m *testing.M) { + os.Exit(m.Run()) +} + +func makeTmpFiles(baseDir string, files []string) { + for _, file := range files { + file = filepath.Join(baseDir, file) + dir, _ := filepath.Split(file) + os.MkdirAll(dir, os.ModePerm) + os.OpenFile(file, os.O_RDONLY|os.O_CREATE, 0666) + } +} + +func checkFiles(baseDir string, resultFiles []string, expectedFiles []string) bool { + var expected []string + for _, file := range expectedFiles { + expected = append(expected, filepath.Join(baseDir, file)) + } + sort.Sort(sort.Reverse(sort.StringSlice(resultFiles))) + sort.Sort(sort.Reverse(sort.StringSlice(expected))) + return !reflect.DeepEqual(resultFiles, expected) +} From d9136ae9822c69358869bba10959e13546d26a28 Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Wed, 15 Dec 2021 21:01:17 -0500 Subject: [PATCH 2/6] Use doublestar in e2e tests now --- cli/scripts/monorepo.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/scripts/monorepo.ts b/cli/scripts/monorepo.ts index fa3f28780c970..7e3056655deea 100644 --- a/cli/scripts/monorepo.ts +++ b/cli/scripts/monorepo.ts @@ -141,7 +141,7 @@ importers: version: "0.1.0", private: true, license: "MIT", - workspaces: ["packages/*"], + workspaces: ["packages/**"], scripts: { build: `${turboPath} run build`, test: `${turboPath} run test`, From 86691a851c9ca15324f9b615342b612695fe8b58 Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Wed, 15 Dec 2021 21:08:17 -0500 Subject: [PATCH 3/6] Use filepath.FromSlash for windows --- cli/internal/fs/globby/globby_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/internal/fs/globby/globby_test.go b/cli/internal/fs/globby/globby_test.go index 7be102397d872..654fdcee01c01 100644 --- a/cli/internal/fs/globby/globby_test.go +++ b/cli/internal/fs/globby/globby_test.go @@ -305,7 +305,7 @@ func TestMain(m *testing.M) { func makeTmpFiles(baseDir string, files []string) { for _, file := range files { - file = filepath.Join(baseDir, file) + file = filepath.Join(baseDir, filepath.FromSlash(file)) dir, _ := filepath.Split(file) os.MkdirAll(dir, os.ModePerm) os.OpenFile(file, os.O_RDONLY|os.O_CREATE, 0666) @@ -315,7 +315,7 @@ func makeTmpFiles(baseDir string, files []string) { func checkFiles(baseDir string, resultFiles []string, expectedFiles []string) bool { var expected []string for _, file := range expectedFiles { - expected = append(expected, filepath.Join(baseDir, file)) + expected = append(expected, filepath.Join(baseDir, filepath.FromSlash(file))) } sort.Sort(sort.Reverse(sort.StringSlice(resultFiles))) sort.Sort(sort.Reverse(sort.StringSlice(expected))) From d5e170c25d77808c91e3408370a526269ea77cc7 Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Wed, 15 Dec 2021 21:09:12 -0500 Subject: [PATCH 4/6] Fix slashes for windows --- cli/internal/fs/globby/globby_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cli/internal/fs/globby/globby_test.go b/cli/internal/fs/globby/globby_test.go index 654fdcee01c01..3982c0e5a16df 100644 --- a/cli/internal/fs/globby/globby_test.go +++ b/cli/internal/fs/globby/globby_test.go @@ -14,7 +14,7 @@ import ( func TestIgnoreDotGitFiles(t *testing.T) { // Init test files curDir, _ := os.Getwd() - tmpDir := filepath.Join(curDir, "./tmp") + tmpDir := filepath.Join(curDir, "tmp") defer os.RemoveAll(tmpDir) makeTmpFiles(tmpDir, []string{ ".git/file", @@ -37,7 +37,7 @@ func TestIgnoreDotGitFiles(t *testing.T) { func TestMathAllImg(t *testing.T) { // Init test files curDir, _ := os.Getwd() - tmpDir := filepath.Join(curDir, "./tmp") + tmpDir := filepath.Join(curDir, "tmp") defer os.RemoveAll(tmpDir) makeTmpFiles(tmpDir, []string{ "app.js", @@ -67,7 +67,7 @@ func TestMathAllImg(t *testing.T) { func TestSignleStarFiles(t *testing.T) { // Init test files curDir, _ := os.Getwd() - tmpDir := filepath.Join(curDir, "./tmp") + tmpDir := filepath.Join(curDir, "tmp") defer os.RemoveAll(tmpDir) makeTmpFiles(tmpDir, []string{ ".git", @@ -102,7 +102,7 @@ func TestSignleStarFiles(t *testing.T) { func TestDirMatch(t *testing.T) { // Init test files curDir, _ := os.Getwd() - tmpDir := filepath.Join(curDir, "./tmp") + tmpDir := filepath.Join(curDir, "tmp") defer os.RemoveAll(tmpDir) makeTmpFiles(tmpDir, []string{ ".git", @@ -138,7 +138,7 @@ func TestDirMatch(t *testing.T) { func TestDirStar(t *testing.T) { // Init test files curDir, _ := os.Getwd() - tmpDir := filepath.Join(curDir, "./tmp") + tmpDir := filepath.Join(curDir, "tmp") defer os.RemoveAll(tmpDir) makeTmpFiles(tmpDir, []string{ ".git", @@ -176,7 +176,7 @@ func TestDirStar(t *testing.T) { func TestDirStar2(t *testing.T) { // Init test files curDir, _ := os.Getwd() - tmpDir := filepath.Join(curDir, "./tmp") + tmpDir := filepath.Join(curDir, "tmp") defer os.RemoveAll(tmpDir) makeTmpFiles(tmpDir, []string{ ".git", @@ -216,7 +216,7 @@ func TestDirStar2(t *testing.T) { func TestDirIgnoreFile(t *testing.T) { // Init test files curDir, _ := os.Getwd() - tmpDir := filepath.Join(curDir, "./tmp") + tmpDir := filepath.Join(curDir, "tmp") defer os.RemoveAll(tmpDir) makeTmpFiles(tmpDir, []string{ ".git", @@ -262,7 +262,7 @@ func TestDirIgnoreFile(t *testing.T) { func TestDirIgnoreDir(t *testing.T) { // Init test files curDir, _ := os.Getwd() - tmpDir := filepath.Join(curDir, "./tmp") + tmpDir := filepath.Join(curDir, "tmp") defer os.RemoveAll(tmpDir) makeTmpFiles(tmpDir, []string{ ".git", From 287129ea572c4faf7e9f14033bec01f21a257a10 Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Thu, 16 Dec 2021 15:15:16 -0500 Subject: [PATCH 5/6] New globby implementation --- cli/go.mod | 2 +- cli/go.sum | 4 +- cli/internal/context/context.go | 30 +-- cli/internal/fs/fs.go | 25 -- cli/internal/fs/globby/globby.go | 206 +++++----------- cli/internal/fs/globby/globby_test.go | 323 -------------------------- cli/internal/run/run.go | 20 +- 7 files changed, 71 insertions(+), 539 deletions(-) delete mode 100644 cli/internal/fs/globby/globby_test.go diff --git a/cli/go.mod b/cli/go.mod index dd0b146eebd47..34ff75c130d28 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -6,7 +6,7 @@ require ( github.com/AlecAivazis/survey/v2 v2.2.12 github.com/adrg/xdg v0.3.3 github.com/armon/go-radix v1.0.0 // indirect - github.com/bmatcuk/doublestar v1.3.4 + github.com/bmatcuk/doublestar/v4 v4.0.2 github.com/briandowns/spinner v1.16.0 github.com/deckarep/golang-set v1.7.1 github.com/fatih/color v1.7.0 diff --git a/cli/go.sum b/cli/go.sum index e474a9957265d..7d793cfd72e47 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -15,8 +15,8 @@ github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= -github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= +github.com/bmatcuk/doublestar/v4 v4.0.2 h1:X0krlUVAVmtr2cRoTqR8aDMrDqnB36ht8wpWTiQ3jsA= +github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/briandowns/spinner v1.16.0 h1:DFmp6hEaIx2QXXuqSJmtfSBSAjRmpGiKG6ip2Wm/yOs= github.com/briandowns/spinner v1.16.0/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FScO+3/ZPQ= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index aa3bd32037c40..3cc307a6d7d01 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -14,7 +14,6 @@ import ( "turbo/internal/fs/globby" "turbo/internal/util" - "github.com/bmatcuk/doublestar" mapset "github.com/deckarep/golang-set" "github.com/fatih/color" "github.com/google/chrometracing" @@ -181,14 +180,9 @@ func WithGraph(rootpath string, config *config.Config) Option { globalDeps := make(util.Set) if len(pkg.Turbo.GlobalDependencies) > 0 { - for _, value := range pkg.Turbo.GlobalDependencies { - f, err := doublestar.Glob(value) - if err != nil { - return fmt.Errorf("error parsing global dependencies glob %v: %w", value, err) - } - for _, val := range f { - globalDeps.Add(val) - } + f := globby.GlobFiles(rootpath, &pkg.Turbo.GlobalDependencies, nil) + for _, val := range f { + globalDeps.Add(val) } } if c.Backend.Name != "nodejs-yarn" || fs.CheckIfWindows() { @@ -242,16 +236,14 @@ func WithGraph(rootpath string, config *config.Config) Option { for i, space := range spaces { justJsons[i] = path.Join(space, "package.json") } - f := globby.Match(justJsons, globby.Option{ - BaseDir: rootpath, - CheckDot: true, - Excludes: []string{ - "**/node_modules/**/*", - "**/bower_components/**/*", - "**/test/**/*", - "**/tests/**/*", - }, - }) + ignore := []string{ + "**/node_modules/**/*", + "**/bower_components/**/*", + "**/test/**/*", + "**/tests/**/*", + } + + f := globby.GlobFiles(rootpath, &justJsons, &ignore) for i, val := range f { _, val := i, val // https://golang.org/doc/faq#closures_and_goroutines diff --git a/cli/internal/fs/fs.go b/cli/internal/fs/fs.go index 377a23d40765f..20e7970dd21ea 100644 --- a/cli/internal/fs/fs.go +++ b/cli/internal/fs/fs.go @@ -1,16 +1,11 @@ package fs import ( - "fmt" "io" "io/ioutil" "log" "os" "path/filepath" - "strings" - "turbo/internal/util" - - "github.com/bmatcuk/doublestar" ) // https://github.com/thought-machine/please/blob/master/src/fs/fs.go @@ -161,23 +156,3 @@ func copyFile(from, to string) (err error) { return nil } - -// GlobList accepts a list of doublestar directive globs and returns a list of files matching them -func Globby(globs []string) ([]string, error) { - var fileset = make(util.Set) - for _, output := range globs { - results, err := doublestar.Glob(strings.TrimPrefix(output, "!")) - if err != nil { - return nil, fmt.Errorf("invalid glob %v: %w", output, err) - } - // we handle negation via "!" by removing the result from the fileset - for _, result := range results { - if strings.HasPrefix(output, "!") { - fileset.Delete(result) - } else { - fileset.Add(result) - } - } - } - return fileset.UnsafeListOfStrings(), nil -} diff --git a/cli/internal/fs/globby/globby.go b/cli/internal/fs/globby/globby.go index e7f14c0550d38..379f493fcd1c1 100644 --- a/cli/internal/fs/globby/globby.go +++ b/cli/internal/fs/globby/globby.go @@ -2,178 +2,78 @@ package globby import ( "fmt" - "os" + "io/fs" "path/filepath" - "regexp" + "strings" + + "github.com/bmatcuk/doublestar/v4" ) -type Option struct { - BaseDir string - CheckDot bool - RelativeReturn bool - Excludes []string -} +// // GlobList accepts a list of doublestar directive globs and returns a list of files matching them +// func Globby(base string, globs []string) ([]string, error) { +// ignoreList := []string{} +// actualGlobs := []string{} +// for _, output := range globs { +// if strings.HasPrefix(output, "!") { +// ignoreList = append(ignoreList, strings.TrimPrefix(output, "!")) +// } else { +// actualGlobs = append(actualGlobs, output) +// } +// } +// files := []string{} +// for _, glob := range actualGlobs { +// matches, err := doublestar.Glob(os.DirFS(base), glob) +// if err != nil { +// return nil, err +// } +// for _, match := range matches { +// for _, ignore := range ignoreList { +// if isMatch, _ := doublestar.PathMatch(ignore, match); !isMatch { +// files = append(files, match) +// } +// } +// } +// } +// } -/* - * Glob all patterns - */ -func Match(patterns []string, opt Option) []string { - var allFiles []string - patterns, opt, err := completeOpt(patterns, opt) - if err != nil { - fmt.Printf("Magth err: [%v]\n", err) - return allFiles - } - for _, pattern := range patterns { - files := find(pattern, opt) - if files == nil || len(*files) == 0 { - continue - } - allFiles = append(allFiles, *files...) - } - return allFiles -} +func GlobFiles(ws_path string, include_pattens *[]string, exclude_pattens *[]string) []string { + var include []string + var exclude []string + var result []string -func find(pattern string, opt Option) *[]string { - // match ./some/path/**/* - if regexTest("\\*\\*", pattern) || - !regexTest("\\*", pattern) { // Dirname - return findRecr(pattern, opt) + for _, p := range *include_pattens { + include = append(include, filepath.Join(ws_path, p)) } - // match ./some/path/* - if regexTest("\\*", pattern) { - return findDir(pattern, opt) - } - return nil -} -// find under centain directory -func findDir(pattern string, opt Option) *[]string { - var list []string - files, err := filepath.Glob(pattern) - if err != nil { - fmt.Printf("err: [%v]\n", err) - return &list + for _, p := range *exclude_pattens { + exclude = append(exclude, filepath.Join(ws_path, p)) } - for _, fullpath := range files { - path, err := filepath.Rel(opt.BaseDir, fullpath) + + var include_pattern = "{" + strings.Join(include, ",") + "}" + var exclude_pattern = "{" + strings.Join(exclude, ",") + "}" + var _ = filepath.Walk(ws_path, func(p string, info fs.FileInfo, err error) error { if err != nil { - continue - } - if checkExclude(opt, path) { - continue - } - if opt.RelativeReturn { - list = append(list, path) - } else { - list = append(list, fullpath) + fmt.Printf("prevent panic by handling failure accessing a path %q: %v\n", p, err) + return err } - } - return &list -} - -// find recursively -func findRecr(pattern string, opt Option) *[]string { - dir := strReplace(pattern, "\\*\\*.+", "") - afterMacth := "" - matchAfterFlag := false - if regexTest("\\*", pattern) { - afterMacth = strReplace(pattern, ".+\\*", "") - matchAfterFlag = len(afterMacth) > 0 - } - var list []string - err := filepath.Walk(dir, func(fullpath string, f os.FileInfo, err error) error { - if !opt.CheckDot && regexTest("^\\.", f.Name()) { - if f.IsDir() { + if val, _ := doublestar.Match(exclude_pattern, p); val { + if info.IsDir() { return filepath.SkipDir } return nil } - if f.IsDir() { - return nil - } - path, _ := filepath.Rel(opt.BaseDir, fullpath) - if checkExclude(opt, path) { - return nil - } - if !opt.RelativeReturn { - path = fullpath - } - if !matchAfterFlag { - list = append(list, path) - return nil - } - if regexTest(afterMacth+"$", path) { - list = append(list, path) - } - return nil - }) - if err != nil { - fmt.Printf("err: [%v]\n", err) - } - return &list -} -// check and complete the options -func completeOpt(srcPatterns []string, opt Option) ([]string, Option, error) { - if len(opt.BaseDir) == 0 { - curDir, err := os.Getwd() - if err != nil { - panic(err) - } - opt.BaseDir = curDir - } - - var patterns []string - for _, pattern := range srcPatterns { - // TODO: check no "tmp/*", use "tmp" or "tmp/*.ext" instead - - if regexTest("^\\!", pattern) { - opt.Excludes = append(opt.Excludes, strReplace(pattern, "^\\!", "")) - continue - } - if regexTest("^\\.", pattern) || // like ./dist - !regexTest("^\\/", pattern) { // like dist - patterns = append(patterns, filepath.Join(opt.BaseDir, pattern)) - continue + if info.IsDir() { + return nil } - patterns = append(patterns, pattern) - } - return patterns, opt, nil -} -// check if path should be excluded -func checkExclude(opt Option, path string) bool { - // if exludes dirs - for _, exclude := range opt.Excludes { - rule := exclude - if regexTest("\\*\\*", exclude) { - rule = strReplace(exclude, "\\*\\*/\\*+?", ".+") - } else if regexTest("\\*", exclude) { - rule = strReplace(exclude, "\\*", "[^/]+") + if val, _ := doublestar.Match(include_pattern, p); val || len(*include_pattens) == 0 { + result = append(result, p) } - if regexTest("^"+rule, path) { - return true // ignore - } - } - return false -} -// Check if regex match the "src" string -func regexTest(re string, src string) bool { - matched, err := regexp.MatchString(re, src) - if err != nil { - return false - } - if matched { - return true - } - return false -} + return nil + }) -// "dest" replace "text" pattern with "repl" -func strReplace(dest, text, repl string) string { - re := regexp.MustCompile(text) - return re.ReplaceAllString(dest, repl) + return result } diff --git a/cli/internal/fs/globby/globby_test.go b/cli/internal/fs/globby/globby_test.go deleted file mode 100644 index 3982c0e5a16df..0000000000000 --- a/cli/internal/fs/globby/globby_test.go +++ /dev/null @@ -1,323 +0,0 @@ -package globby - -import ( - "os" - "path/filepath" - "reflect" - "sort" - "testing" -) - -/* - * Test ignore .git - */ -func TestIgnoreDotGitFiles(t *testing.T) { - // Init test files - curDir, _ := os.Getwd() - tmpDir := filepath.Join(curDir, "tmp") - defer os.RemoveAll(tmpDir) - makeTmpFiles(tmpDir, []string{ - ".git/file", - ".gitignore", - "app.js", - }) - - // Match the patterns - files := Match([]string{"."}, Option{BaseDir: tmpDir}) - // Expected match files: - expected := []string{"app.js"} - if checkFiles(tmpDir, files, expected) { - t.Errorf("files not match, expected %v, but got %v", expected, files) - } -} - -/* - * Match "./** /*.jpg" - */ -func TestMathAllImg(t *testing.T) { - // Init test files - curDir, _ := os.Getwd() - tmpDir := filepath.Join(curDir, "tmp") - defer os.RemoveAll(tmpDir) - makeTmpFiles(tmpDir, []string{ - "app.js", - "src/test.js", - "image/footer.jpg", - "image/logo.jpg", - "image/user/avatar.jpg", - }) - // Match the patterns - files := Match([]string{ - "./**/*.jpg", - }, Option{BaseDir: tmpDir}) - // Expected match files: - expected := []string{ - "image/footer.jpg", - "image/logo.jpg", - "image/user/avatar.jpg", - } - if checkFiles(tmpDir, files, expected) { - t.Errorf("files not match, expected %v, but got %v", expected, files) - } -} - -/* - * Match "src/*.js" - */ -func TestSignleStarFiles(t *testing.T) { - // Init test files - curDir, _ := os.Getwd() - tmpDir := filepath.Join(curDir, "tmp") - defer os.RemoveAll(tmpDir) - makeTmpFiles(tmpDir, []string{ - ".git", - "app.js", - "package.json", - "src/router.js", - "src/store.js", - "src/api/home.js", - "src/api/user.js", - "src/api/test.js", - }) - - patterns := []string{ - "src/*.js", - } - - // Match the patterns - files := Match(patterns, Option{BaseDir: tmpDir}) - // Expected match files: - expected := []string{ - "src/router.js", - "src/store.js", - } - if checkFiles(tmpDir, files, expected) { - t.Errorf("files not match, expected %v, but got %v", expected, files) - } -} - -/* - * Match "src/api" - */ -func TestDirMatch(t *testing.T) { - // Init test files - curDir, _ := os.Getwd() - tmpDir := filepath.Join(curDir, "tmp") - defer os.RemoveAll(tmpDir) - makeTmpFiles(tmpDir, []string{ - ".git", - "app.js", - "package.json", - "src/router.js", - "src/store.js", - "src/api/home.js", - "src/api/user.js", - "src/api/test.js", - }) - - patterns := []string{ - "src/api", - } - - // Match the patterns - files := Match(patterns, Option{BaseDir: tmpDir}) - // Expected match files: - expected := []string{ - "src/api/home.js", - "src/api/user.js", - "src/api/test.js", - } - if checkFiles(tmpDir, files, expected) { - t.Errorf("files not match, expected %v, but got %v", expected, files) - } -} - -/* - * Match "/**" + "/*" - */ -func TestDirStar(t *testing.T) { - // Init test files - curDir, _ := os.Getwd() - tmpDir := filepath.Join(curDir, "tmp") - defer os.RemoveAll(tmpDir) - makeTmpFiles(tmpDir, []string{ - ".git", - "app.js", - "package.json", - "src/router.js", - "src/store.js", - "src/api/home.js", - "src/api/user.js", - "src/api/test.js", - }) - - patterns := []string{ - "src/**/*", - } - - // Match the patterns - files := Match(patterns, Option{BaseDir: tmpDir}) - // Expected match files: - expected := []string{ - "src/router.js", - "src/store.js", - "src/api/home.js", - "src/api/user.js", - "src/api/test.js", - } - if checkFiles(tmpDir, files, expected) { - t.Errorf("files not match, expected %v, but got %v", expected, files) - } -} - -/* - * Match "/**" + "/*.js" - */ -func TestDirStar2(t *testing.T) { - // Init test files - curDir, _ := os.Getwd() - tmpDir := filepath.Join(curDir, "tmp") - defer os.RemoveAll(tmpDir) - makeTmpFiles(tmpDir, []string{ - ".git", - "app.js", - "package.json", - "src/router.js", - "src/store.js", - "src/store.ts", - "src/api/home.js", - "src/api/user.js", - "src/api/test.js", - }) - - patterns := []string{ - "src/**/*.js", - } - - // Match the patterns - files := Match(patterns, Option{BaseDir: tmpDir}) - // Expected match files: - expected := []string{ - "src/router.js", - "src/store.js", - "src/api/home.js", - "src/api/user.js", - "src/api/test.js", - } - if checkFiles(tmpDir, files, expected) { - t.Errorf("files not match, expected %v, but got %v", expected, files) - } -} - -/* - * Match "/**" + "/*.js" - * ignore files in the match items - */ -func TestDirIgnoreFile(t *testing.T) { - // Init test files - curDir, _ := os.Getwd() - tmpDir := filepath.Join(curDir, "tmp") - defer os.RemoveAll(tmpDir) - makeTmpFiles(tmpDir, []string{ - ".git", - "app.js", - "package.json", - "src/router.js", - "src/store.js", - "src/store.ts", - "src/api/home.js", - "src/api/user.js", - "src/api/test.js", - "src/service/home.js", - "src/service/user.js", - "src/service/test.js", - }) - - patterns := []string{ - "src/**/*.js", - "!src/service/home.js", - } - - // Match the patterns - files := Match(patterns, Option{BaseDir: tmpDir}) - // Expected match files: - expected := []string{ - "src/router.js", - "src/store.js", - "src/api/home.js", - "src/api/user.js", - "src/api/test.js", - "src/service/user.js", - "src/service/test.js", - } - if checkFiles(tmpDir, files, expected) { - t.Errorf("files not match, expected %v, but got %v", expected, files) - } -} - -/* - * Match "/**" + "/*.js" - * ignore dir in the match items - */ -func TestDirIgnoreDir(t *testing.T) { - // Init test files - curDir, _ := os.Getwd() - tmpDir := filepath.Join(curDir, "tmp") - defer os.RemoveAll(tmpDir) - makeTmpFiles(tmpDir, []string{ - ".git", - "app.js", - "package.json", - "src/router.js", - "src/store.js", - "src/store.ts", - "src/api/home.js", - "src/api/user.js", - "src/api/test.js", - "src/service/home.js", - "src/service/user.js", - "src/service/test.js", - }) - - patterns := []string{ - "src/**/*.js", - "!src/service", - } - - // Match the patterns - files := Match(patterns, Option{BaseDir: tmpDir}) - // Expected match files: - expected := []string{ - "src/router.js", - "src/store.js", - "src/api/home.js", - "src/api/user.js", - "src/api/test.js", - } - if checkFiles(tmpDir, files, expected) { - t.Errorf("files not match, expected %v, but got %v", expected, files) - } -} - -func TestMain(m *testing.M) { - os.Exit(m.Run()) -} - -func makeTmpFiles(baseDir string, files []string) { - for _, file := range files { - file = filepath.Join(baseDir, filepath.FromSlash(file)) - dir, _ := filepath.Split(file) - os.MkdirAll(dir, os.ModePerm) - os.OpenFile(file, os.O_RDONLY|os.O_CREATE, 0666) - } -} - -func checkFiles(baseDir string, resultFiles []string, expectedFiles []string) bool { - var expected []string - for _, file := range expectedFiles { - expected = append(expected, filepath.Join(baseDir, filepath.FromSlash(file))) - } - sort.Sort(sort.Reverse(sort.StringSlice(resultFiles))) - sort.Sort(sort.Reverse(sort.StringSlice(expected))) - return !reflect.DeepEqual(resultFiles, expected) -} diff --git a/cli/internal/run/run.go b/cli/internal/run/run.go index 2439f7b4c963d..de66df041595c 100644 --- a/cli/internal/run/run.go +++ b/cli/internal/run/run.go @@ -20,11 +20,11 @@ import ( "turbo/internal/context" "turbo/internal/core" "turbo/internal/fs" + "turbo/internal/fs/globby" "turbo/internal/scm" "turbo/internal/ui" "turbo/internal/util" - "github.com/bmatcuk/doublestar" "github.com/pyr-sh/dag" "github.com/fatih/color" @@ -582,21 +582,9 @@ func (c *RunCommand) Run(args []string) int { if runOptions.cache && (pipeline.Cache == nil || *pipeline.Cache) { targetLogger.Debug("caching output", "outputs", outputs) - var filesToBeCached = make(util.Set) - for _, output := range outputs { - results, err := doublestar.Glob(filepath.Join(pack.Dir, strings.TrimPrefix(output, "!"))) - if err != nil { - targetUi.Error(fmt.Sprintf("Could not find output artifacts in %v, likely invalid glob %v: %s", pack.Dir, output, err)) - } - for _, result := range results { - if strings.HasPrefix(output, "!") { - filesToBeCached.Delete(result) - } else { - filesToBeCached.Add(result) - } - } - } - if err := turboCache.Put(pack.Dir, hash, int(time.Since(cmdTime).Milliseconds()), filesToBeCached.UnsafeListOfStrings()); err != nil { + ignore := []string{} + filesToBeCached := globby.GlobFiles(pack.Dir, &outputs, &ignore) + if err := turboCache.Put(pack.Dir, hash, int(time.Since(cmdTime).Milliseconds()), filesToBeCached); err != nil { c.logError(targetLogger, "", fmt.Errorf("Error caching output: %w", err)) } } From e3c97d60bfe6a59e8cf7df0b7aa8bc8cd10cc7bf Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Thu, 16 Dec 2021 15:25:45 -0500 Subject: [PATCH 6/6] Use pathmatch --- cli/internal/fs/globby/globby.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/internal/fs/globby/globby.go b/cli/internal/fs/globby/globby.go index 379f493fcd1c1..61c0d46c032cb 100644 --- a/cli/internal/fs/globby/globby.go +++ b/cli/internal/fs/globby/globby.go @@ -57,7 +57,7 @@ func GlobFiles(ws_path string, include_pattens *[]string, exclude_pattens *[]str return err } - if val, _ := doublestar.Match(exclude_pattern, p); val { + if val, _ := doublestar.PathMatch(exclude_pattern, p); val { if info.IsDir() { return filepath.SkipDir } @@ -68,7 +68,7 @@ func GlobFiles(ws_path string, include_pattens *[]string, exclude_pattens *[]str return nil } - if val, _ := doublestar.Match(include_pattern, p); val || len(*include_pattens) == 0 { + if val, _ := doublestar.PathMatch(include_pattern, p); val || len(*include_pattens) == 0 { result = append(result, p) }