vercel · jaredpalmer · Dec 16, 2021 · Dec 16, 2021 · Dec 16, 2021 · Dec 16, 2021
@@ -6,7 +6,7 @@ require (
 	github.com/AlecAivazis/survey/v2 v2.2.12
 	github.com/adrg/xdg v0.3.3
 	github.com/armon/go-radix v1.0.0 // indirect
-	github.com/bmatcuk/doublestar v1.3.4
+	github.com/bmatcuk/doublestar/v4 v4.0.2
 	github.com/briandowns/spinner v1.16.0
 	github.com/deckarep/golang-set v1.7.1
 	github.com/fatih/color v1.7.0

@@ -15,8 +15,8 @@ github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=
-github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
+github.com/bmatcuk/doublestar/v4 v4.0.2 h1:X0krlUVAVmtr2cRoTqR8aDMrDqnB36ht8wpWTiQ3jsA=
+github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/briandowns/spinner v1.16.0 h1:DFmp6hEaIx2QXXuqSJmtfSBSAjRmpGiKG6ip2Wm/yOs=
 github.com/briandowns/spinner v1.16.0/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FScO+3/ZPQ=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=

@@ -2,7 +2,7 @@ package context
 
 import (
 	"fmt"
-	"log"
+	"path"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -11,9 +11,9 @@ import (
 	"turbo/internal/backends"
 	"turbo/internal/config"
 	"turbo/internal/fs"
+	"turbo/internal/fs/globby"
 	"turbo/internal/util"
 
-	"github.com/bmatcuk/doublestar"
 	mapset "github.com/deckarep/golang-set"
 	"github.com/fatih/color"
 	"github.com/google/chrometracing"
@@ -180,14 +180,9 @@ func WithGraph(rootpath string, config *config.Config) Option {
 		globalDeps := make(util.Set)
 
 		if len(pkg.Turbo.GlobalDependencies) > 0 {
-			for _, value := range pkg.Turbo.GlobalDependencies {
-				f, err := filepath.Glob(value)
-				if err != nil {
-					return fmt.Errorf("error parsing global dependencies glob %v: %w", value, err)
-				}
-				for _, val := range f {
-					globalDeps.Add(val)
-				}
+			f := globby.GlobFiles(rootpath, &pkg.Turbo.GlobalDependencies, nil)
+			for _, val := range f {
+				globalDeps.Add(val)
 			}
 		}
 		if c.Backend.Name != "nodejs-yarn" || fs.CheckIfWindows() {
@@ -237,18 +232,24 @@ func WithGraph(rootpath string, config *config.Config) Option {
 		// until all parsing is complete
 		// and populate the graph
 		parseJSONWaitGroup := new(errgroup.Group)
-		for _, value := range spaces {
-			f, err := doublestar.Glob(value)
-			if err != nil {
-				log.Fatalf("Error parsing workspaces glob %v", value)
-			}
+		justJsons := make([]string, len(spaces))
+		for i, space := range spaces {
+			justJsons[i] = path.Join(space, "package.json")
+		}
+		ignore := []string{
+			"**/node_modules/**/*",
+			"**/bower_components/**/*",
+			"**/test/**/*",
+			"**/tests/**/*",
+		}
 
-			for i, val := range f {
-				_, val := i, val // https://golang.org/doc/faq#closures_and_goroutines
-				parseJSONWaitGroup.Go(func() error {
-					return c.parsePackageJSON(val)
-				})
-			}
+		f := globby.GlobFiles(rootpath, &justJsons, &ignore)
+
+		for i, val := range f {
+			_, val := i, val // https://golang.org/doc/faq#closures_and_goroutines
+			parseJSONWaitGroup.Go(func() error {
+				return c.parsePackageJSON(val)
+			})
 		}
 
 		if err := parseJSONWaitGroup.Wait(); err != nil {
@@ -442,10 +443,9 @@ func (c *Context) populateTopologicGraphForPackageJson(pkg *fs.PackageJSON) erro
 	return nil
 }
 
-func (c *Context) parsePackageJSON(fileName string) error {
+func (c *Context) parsePackageJSON(buildFilePath string) error {
 	c.mutex.Lock()
 	defer c.mutex.Unlock()
-	buildFilePath := filepath.Join(fileName, "package.json")
 
 	// log.Printf("[TRACE] reading package.json : %+v", buildFilePath)
 	if fs.FileExists(buildFilePath) {
@@ -457,7 +457,7 @@ func (c *Context) parsePackageJSON(fileName string) error {
 		// log.Printf("[TRACE] adding %+v to graph", pkg.Name)
 		c.TopologicalGraph.Add(pkg.Name)
 		pkg.PackageJSONPath = buildFilePath
-		pkg.Dir = fileName
+		pkg.Dir = filepath.Dir(buildFilePath)
 		c.PackageInfos[pkg.Name] = pkg
 		c.PackageNames = append(c.PackageNames, pkg.Name)
 	}

@@ -1,16 +1,11 @@
 package fs
 
 import (
-	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
-	"strings"
-	"turbo/internal/util"
-
-	"github.com/bmatcuk/doublestar"
 )
 
 // https://github.com/thought-machine/please/blob/master/src/fs/fs.go
@@ -161,23 +156,3 @@ func copyFile(from, to string) (err error) {
 
 	return nil
 }
-
-// GlobList accepts a list of doublestar directive globs and returns a list of files matching them
-func Globby(globs []string) ([]string, error) {
-	var fileset = make(util.Set)
-	for _, output := range globs {
-		results, err := doublestar.Glob(strings.TrimPrefix(output, "!"))
-		if err != nil {
-			return nil, fmt.Errorf("invalid glob %v: %w", output, err)
-		}
-		// we handle negation via "!" by removing the result from the fileset
-		for _, result := range results {
-			if strings.HasPrefix(output, "!") {
-				fileset.Delete(result)
-			} else {
-				fileset.Add(result)
-			}
-		}
-	}
-	return fileset.UnsafeListOfStrings(), nil
-}
@@ -0,0 +1,79 @@
+package globby
+
+import (
+	"fmt"
+	"io/fs"
+	"path/filepath"
+	"strings"
+
+	"github.com/bmatcuk/doublestar/v4"
+)
+
+// // GlobList accepts a list of doublestar directive globs and returns a list of files matching them
+// func Globby(base string, globs []string) ([]string, error) {
+// 	ignoreList := []string{}
+// 	actualGlobs := []string{}
+// 	for _, output := range globs {
+// 		if strings.HasPrefix(output, "!") {
+// 			ignoreList = append(ignoreList, strings.TrimPrefix(output, "!"))
+// 		} else {
+// 			actualGlobs = append(actualGlobs, output)
+// 		}
+// 	}
+// 	files := []string{}
+// 	for _, glob := range actualGlobs {
+// 		matches, err := doublestar.Glob(os.DirFS(base), glob)
+// 		if err != nil {
+// 			return nil, err
+// 		}
+// 		for _, match := range matches {
+// 			for _, ignore := range ignoreList {
+// 				if isMatch, _ := doublestar.PathMatch(ignore, match); !isMatch {
+// 					files = append(files, match)
+// 				}
+// 			}
+// 		}
+// 	}
+// }
+
+func GlobFiles(ws_path string, include_pattens *[]string, exclude_pattens *[]string) []string {
+	var include []string
+	var exclude []string
+	var result []string
+
+	for _, p := range *include_pattens {
+		include = append(include, filepath.Join(ws_path, p))
+	}
+
+	for _, p := range *exclude_pattens {
+		exclude = append(exclude, filepath.Join(ws_path, p))
+	}
+
+	var include_pattern = "{" + strings.Join(include, ",") + "}"
+	var exclude_pattern = "{" + strings.Join(exclude, ",") + "}"
+	var _ = filepath.Walk(ws_path, func(p string, info fs.FileInfo, err error) error {
+		if err != nil {
+			fmt.Printf("prevent panic by handling failure accessing a path %q: %v\n", p, err)
+			return err
+		}
+
+		if val, _ := doublestar.PathMatch(exclude_pattern, p); val {
+			if info.IsDir() {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+
+		if info.IsDir() {
+			return nil
+		}
+
+		if val, _ := doublestar.PathMatch(include_pattern, p); val || len(*include_pattens) == 0 {
+			result = append(result, p)
+		}
+
+		return nil
+	})
+
+	return result
+}
@@ -20,11 +20,11 @@ import (
 	"turbo/internal/context"
 	"turbo/internal/core"
 	"turbo/internal/fs"
+	"turbo/internal/fs/globby"
 	"turbo/internal/scm"
 	"turbo/internal/ui"
 	"turbo/internal/util"
 
-	"github.com/bmatcuk/doublestar"
 	"github.com/pyr-sh/dag"
 
 	"github.com/fatih/color"
@@ -582,21 +582,9 @@ func (c *RunCommand) Run(args []string) int {
 
 				if runOptions.cache && (pipeline.Cache == nil || *pipeline.Cache) {
 					targetLogger.Debug("caching output", "outputs", outputs)
-					var filesToBeCached = make(util.Set)
-					for _, output := range outputs {
-						results, err := doublestar.Glob(filepath.Join(pack.Dir, strings.TrimPrefix(output, "!")))
-						if err != nil {
-							targetUi.Error(fmt.Sprintf("Could not find output artifacts in %v, likely invalid glob %v: %s", pack.Dir, output, err))
-						}
-						for _, result := range results {
-							if strings.HasPrefix(output, "!") {
-								filesToBeCached.Delete(result)
-							} else {
-								filesToBeCached.Add(result)
-							}
-						}
-					}
-					if err := turboCache.Put(pack.Dir, hash, int(time.Since(cmdTime).Milliseconds()), filesToBeCached.UnsafeListOfStrings()); err != nil {
+					ignore := []string{}
+					filesToBeCached := globby.GlobFiles(pack.Dir, &outputs, &ignore)
+					if err := turboCache.Put(pack.Dir, hash, int(time.Since(cmdTime).Milliseconds()), filesToBeCached); err != nil {
 						c.logError(targetLogger, "", fmt.Errorf("Error caching output: %w", err))
 					}
 				}

@@ -141,7 +141,7 @@ importers:
         version: "0.1.0",
         private: true,
         license: "MIT",
-        workspaces: ["packages/*"],
+        workspaces: ["packages/**"],
         scripts: {
           build: `${turboPath} run build`,
           test: `${turboPath} run test`,