From bfd1f335537632bc5fa14269947b64996155050a Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Thu, 16 Dec 2021 18:24:25 -0500 Subject: [PATCH 1/5] Refactor to faster globs --- cli/internal/context/context.go | 2 +- cli/internal/fs/copy_file.go | 9 ++-- cli/internal/fs/globby/globby.go | 79 -------------------------------- cli/internal/globby/globby.go | 45 ++++++++++++++++++ cli/internal/run/run.go | 2 +- 5 files changed, 53 insertions(+), 84 deletions(-) delete mode 100644 cli/internal/fs/globby/globby.go create mode 100644 cli/internal/globby/globby.go diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index 3cc307a6d7d01..32d65d8082f0b 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -11,7 +11,7 @@ import ( "turbo/internal/backends" "turbo/internal/config" "turbo/internal/fs" - "turbo/internal/fs/globby" + "turbo/internal/globby" "turbo/internal/util" mapset "github.com/deckarep/golang-set" diff --git a/cli/internal/fs/copy_file.go b/cli/internal/fs/copy_file.go index d7eaeca703a28..4f89d81cbf536 100644 --- a/cli/internal/fs/copy_file.go +++ b/cli/internal/fs/copy_file.go @@ -85,9 +85,12 @@ func WalkMode(rootPath string, callback func(name string, isDir bool, mode os.Fi } else if !info.IsDir() { return callback(rootPath, false, info.Mode()) } - return godirwalk.Walk(rootPath, &godirwalk.Options{Callback: func(name string, info *godirwalk.Dirent) error { - return callback(name, info.IsDir(), info.ModeType()) - }}) + return godirwalk.Walk(rootPath, &godirwalk.Options{ + Callback: func(name string, info *godirwalk.Dirent) error { + return callback(name, info.IsDir(), info.ModeType()) + }, + Unsorted: true, + }) } // SameFile returns true if the two given paths refer to the same physical diff --git a/cli/internal/fs/globby/globby.go b/cli/internal/fs/globby/globby.go deleted file mode 100644 index 61c0d46c032cb..0000000000000 --- a/cli/internal/fs/globby/globby.go +++ /dev/null @@ -1,79 +0,0 @@ -package globby - -import ( - "fmt" - "io/fs" - "path/filepath" - "strings" - - "github.com/bmatcuk/doublestar/v4" -) - -// // GlobList accepts a list of doublestar directive globs and returns a list of files matching them -// func Globby(base string, globs []string) ([]string, error) { -// ignoreList := []string{} -// actualGlobs := []string{} -// for _, output := range globs { -// if strings.HasPrefix(output, "!") { -// ignoreList = append(ignoreList, strings.TrimPrefix(output, "!")) -// } else { -// actualGlobs = append(actualGlobs, output) -// } -// } -// files := []string{} -// for _, glob := range actualGlobs { -// matches, err := doublestar.Glob(os.DirFS(base), glob) -// if err != nil { -// return nil, err -// } -// for _, match := range matches { -// for _, ignore := range ignoreList { -// if isMatch, _ := doublestar.PathMatch(ignore, match); !isMatch { -// files = append(files, match) -// } -// } -// } -// } -// } - -func GlobFiles(ws_path string, include_pattens *[]string, exclude_pattens *[]string) []string { - var include []string - var exclude []string - var result []string - - for _, p := range *include_pattens { - include = append(include, filepath.Join(ws_path, p)) - } - - for _, p := range *exclude_pattens { - exclude = append(exclude, filepath.Join(ws_path, p)) - } - - var include_pattern = "{" + strings.Join(include, ",") + "}" - var exclude_pattern = "{" + strings.Join(exclude, ",") + "}" - var _ = filepath.Walk(ws_path, func(p string, info fs.FileInfo, err error) error { - if err != nil { - fmt.Printf("prevent panic by handling failure accessing a path %q: %v\n", p, err) - return err - } - - if val, _ := doublestar.PathMatch(exclude_pattern, p); val { - if info.IsDir() { - return filepath.SkipDir - } - return nil - } - - if info.IsDir() { - return nil - } - - if val, _ := doublestar.PathMatch(include_pattern, p); val || len(*include_pattens) == 0 { - result = append(result, p) - } - - return nil - }) - - return result -} diff --git a/cli/internal/globby/globby.go b/cli/internal/globby/globby.go new file mode 100644 index 0000000000000..12fcc852392d1 --- /dev/null +++ b/cli/internal/globby/globby.go @@ -0,0 +1,45 @@ +package globby + +import ( + "turbo/internal/fs" + + "path/filepath" + "strings" + + "github.com/bmatcuk/doublestar/v4" + "github.com/karrick/godirwalk" +) + +func GlobFiles(ws_path string, include_pattens *[]string, exclude_pattens *[]string) []string { + var include []string + var exclude []string + var result []string + + for _, p := range *include_pattens { + include = append(include, filepath.Join(ws_path, p)) + } + + for _, p := range *exclude_pattens { + exclude = append(exclude, filepath.Join(ws_path, p)) + } + + var include_pattern = "{" + strings.Join(include, ",") + "}" + var exclude_pattern = "{" + strings.Join(exclude, ",") + "}" + var _ = fs.Walk(ws_path, func(p string, isDir bool) error { + if val, _ := doublestar.PathMatch(exclude_pattern, p); val { + return godirwalk.SkipThis + } + + if isDir { + return nil + } + + if val, _ := doublestar.PathMatch(include_pattern, p); val || len(*include_pattens) == 0 { + result = append(result, p) + } + + return nil + }) + + return result +} diff --git a/cli/internal/run/run.go b/cli/internal/run/run.go index de66df041595c..4ae8a93f0adc6 100644 --- a/cli/internal/run/run.go +++ b/cli/internal/run/run.go @@ -20,7 +20,7 @@ import ( "turbo/internal/context" "turbo/internal/core" "turbo/internal/fs" - "turbo/internal/fs/globby" + "turbo/internal/globby" "turbo/internal/scm" "turbo/internal/ui" "turbo/internal/util" From d7f04cd1a0f15bdf104a15b0b5a6ad138c85998e Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Thu, 16 Dec 2021 19:52:10 -0500 Subject: [PATCH 2/5] tesst --- cli/internal/globby/globby.go | 14 +- cli/internal/globby/globby_test.go | 40 +++ .../globby/testdata/foo/package_deps_hash.go | 290 ++++++++++++++++++ .../testdata/foo/package_deps_hash_test.go | 85 +++++ .../globby/testdata/foo/package_json.go | 115 +++++++ .../globby/testdata/package_deps_hash.go | 290 ++++++++++++++++++ .../globby/testdata/package_deps_hash_test.go | 85 +++++ cli/internal/globby/testdata/package_json.go | 115 +++++++ 8 files changed, 1028 insertions(+), 6 deletions(-) create mode 100644 cli/internal/globby/globby_test.go create mode 100644 cli/internal/globby/testdata/foo/package_deps_hash.go create mode 100644 cli/internal/globby/testdata/foo/package_deps_hash_test.go create mode 100644 cli/internal/globby/testdata/foo/package_json.go create mode 100644 cli/internal/globby/testdata/package_deps_hash.go create mode 100644 cli/internal/globby/testdata/package_deps_hash_test.go create mode 100644 cli/internal/globby/testdata/package_json.go diff --git a/cli/internal/globby/globby.go b/cli/internal/globby/globby.go index 12fcc852392d1..be92752d03362 100644 --- a/cli/internal/globby/globby.go +++ b/cli/internal/globby/globby.go @@ -7,19 +7,18 @@ import ( "strings" "github.com/bmatcuk/doublestar/v4" - "github.com/karrick/godirwalk" ) -func GlobFiles(ws_path string, include_pattens *[]string, exclude_pattens *[]string) []string { +func GlobFiles(ws_path string, include_pattens []string, exclude_pattens []string) []string { var include []string var exclude []string var result []string - for _, p := range *include_pattens { + for _, p := range include_pattens { include = append(include, filepath.Join(ws_path, p)) } - for _, p := range *exclude_pattens { + for _, p := range exclude_pattens { exclude = append(exclude, filepath.Join(ws_path, p)) } @@ -27,14 +26,17 @@ func GlobFiles(ws_path string, include_pattens *[]string, exclude_pattens *[]str var exclude_pattern = "{" + strings.Join(exclude, ",") + "}" var _ = fs.Walk(ws_path, func(p string, isDir bool) error { if val, _ := doublestar.PathMatch(exclude_pattern, p); val { - return godirwalk.SkipThis + if isDir { + return filepath.SkipDir + } + return nil } if isDir { return nil } - if val, _ := doublestar.PathMatch(include_pattern, p); val || len(*include_pattens) == 0 { + if val, _ := doublestar.PathMatch(include_pattern, p); val || len(include_pattens) == 0 { result = append(result, p) } diff --git a/cli/internal/globby/globby_test.go b/cli/internal/globby/globby_test.go new file mode 100644 index 0000000000000..87adc34bc4e53 --- /dev/null +++ b/cli/internal/globby/globby_test.go @@ -0,0 +1,40 @@ +package globby + +import ( + "reflect" + "testing" +) + +func TestGlobFiles(t *testing.T) { + + type args struct { + ws_path string + include_pattens []string + exclude_pattens []string + } + tests := []struct { + name string + args args + want []string + }{ + { + name: "globFiles", + args: args{ + ws_path: "testdata", + include_pattens: []string{"package/**/*.go"}, + exclude_pattens: []string{"**/node_modules/**"}, + }, + want: []string{ + "package_deps_hash.go", + }, + }, + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GlobFiles(tt.args.ws_path, tt.args.include_pattens, tt.args.exclude_pattens); !reflect.DeepEqual(got, tt.want) { + t.Errorf("GlobFiles() = %#v, want %#v", got, tt.want) + } + }) + } +} diff --git a/cli/internal/globby/testdata/foo/package_deps_hash.go b/cli/internal/globby/testdata/foo/package_deps_hash.go new file mode 100644 index 0000000000000..23af9681b2d16 --- /dev/null +++ b/cli/internal/globby/testdata/foo/package_deps_hash.go @@ -0,0 +1,290 @@ +package fs + +import ( + "bytes" + "fmt" + "os/exec" + "path/filepath" + "regexp" + "strings" + "turbo/internal/util" +) + +// Predefine []byte variables to avoid runtime allocations. +var ( + escapedSlash = []byte(`\\`) + regularSlash = []byte(`\`) + escapedTab = []byte(`\t`) + regularTab = []byte("\t") +) + +// PackageDepsOptions are parameters for getting git hashes for a filesystem +type PackageDepsOptions struct { + // PackagePath is the folder path to derive the package dependencies from. This is typically the folder + // containing package.json. If omitted, the default value is the current working directory. + PackagePath string + // ExcludedPaths is an optional array of file path exclusions. If a file should be omitted from the list + // of dependencies, use this to exclude it. + ExcludedPaths []string + // GitPath is an optional alternative path to the git installation + GitPath string +} + +// GetPackageDeps Builds an object containing git hashes for the files under the specified `packagePath` folder. +func GetPackageDeps(p *PackageDepsOptions) (map[string]string, error) { + gitLsOutput, err := gitLsTree(p.PackagePath, p.GitPath) + if err != nil { + return nil, fmt.Errorf("Could not get git hashes for files in package %s: %w", p.PackagePath, err) + } + // Add all the checked in hashes. + result := parseGitLsTree(gitLsOutput) + + if len(p.ExcludedPaths) > 0 { + for _, p := range p.ExcludedPaths { + // @todo explore optimization + delete(result, p) + } + } + + // Update the checked in hashes with the current repo status + gitStatusOutput, err := gitStatus(p.PackagePath, p.GitPath) + if err != nil { + return nil, err + } + currentlyChangedFiles := parseGitStatus(gitStatusOutput, p.PackagePath) + var filesToHash []string + excludedPathsSet := new(util.Set) + for filename, changeType := range currentlyChangedFiles { + if changeType == "D" || (len(changeType) == 2 && string(changeType)[1] == []byte("D")[0]) { + delete(result, filename) + } else { + if !excludedPathsSet.Include(filename) { + filesToHash = append(filesToHash, filename) + } + } + } + + // log.Printf("[TRACE] %v:", gitStatusOutput) + // log.Printf("[TRACE] start GitHashForFiles") + current, err := GitHashForFiles( + filesToHash, + p.PackagePath, + ) + if err != nil { + return nil, fmt.Errorf("could not retrieve git hash for files in %s", p.PackagePath) + } + // log.Printf("[TRACE] end GitHashForFiles") + // log.Printf("[TRACE] GitHashForFiles files %v", current) + for filename, hash := range current { + // log.Printf("[TRACE] GitHashForFiles files %v: %v", filename, hash) + result[filename] = hash + } + // log.Printf("[TRACE] GitHashForFiles result %v", result) + return result, nil +} + +// GitHashForFiles a list of files returns a map of with their git hash values. It uses +// git hash-object under the +func GitHashForFiles(filesToHash []string, PackagePath string) (map[string]string, error) { + changes := make(map[string]string) + if len(filesToHash) > 0 { + var input = []string{"hash-object"} + + for _, filename := range filesToHash { + input = append(input, filepath.Join(PackagePath, filename)) + } + // fmt.Println(input) + cmd := exec.Command("git", input...) + // https://blog.kowalczyk.info/article/wOYk/advanced-command-execution-in-go-with-osexec.html + cmd.Stdin = strings.NewReader(strings.Join(input, "\n")) + cmd.Dir = PackagePath + out, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("git hash-object exited with status: %w", err) + } + offByOne := strings.Split(string(out), "\n") // there is an extra "" + hashes := offByOne[:len(offByOne)-1] + if len(hashes) != len(filesToHash) { + return nil, fmt.Errorf("passed %v file paths to Git to hash, but received %v hashes.", len(filesToHash), len(hashes)) + } + for i, hash := range hashes { + filepath := filesToHash[i] + changes[filepath] = hash + } + } + + return changes, nil +} + +// UnescapeChars reverses escaped characters. +func UnescapeChars(in []byte) []byte { + if bytes.ContainsAny(in, "\\\t") { + return in + } + + out := bytes.Replace(in, escapedSlash, regularSlash, -1) + out = bytes.Replace(out, escapedTab, regularTab, -1) + return out +} + +// gitLsTree executes "git ls-tree" in a folder +func gitLsTree(path string, gitPath string) (string, error) { + + cmd := exec.Command("git", "ls-tree", "HEAD", "-r") + cmd.Dir = path + out, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("Failed to read `git ls-tree`: %w", err) + } + return strings.TrimSpace(string(out)), nil +} + +func parseGitLsTree(output string) map[string]string { + changes := make(map[string]string) + if len(output) > 0 { + // A line is expected to look like: + // 100644 blob 3451bccdc831cb43d7a70ed8e628dcf9c7f888c8 src/typings/tsd.d.ts + // 160000 commit c5880bf5b0c6c1f2e2c43c95beeb8f0a808e8bac rushstack + gitRex := regexp.MustCompile(`([0-9]{6})\s(blob|commit)\s([a-f0-9]{40})\s*(.*)`) + outputLines := strings.Split(output, "\n") + + for _, line := range outputLines { + if len(line) > 0 { + matches := gitRex.MatchString(line) + if matches == true { + // this looks like this + // [["160000 commit c5880bf5b0c6c1f2e2c43c95beeb8f0a808e8bac rushstack" "160000" "commit" "c5880bf5b0c6c1f2e2c43c95beeb8f0a808e8bac" "rushstack"]] + match := gitRex.FindAllStringSubmatch(line, -1) + if len(match[0][3]) > 0 && len(match[0][4]) > 0 { + hash := match[0][3] + filename := parseGitFilename(match[0][4]) + changes[filename] = hash + } + // @todo error + } + } + } + } + return changes +} + +// Couldn't figure out how to deal with special characters. Skipping for now. +// @todo see https://github.com/microsoft/rushstack/blob/925ad8c9e22997c1edf5fe38c53fa618e8180f70/libraries/package-deps-hash/src/getPackageDeps.ts#L19 +func parseGitFilename(filename string) string { + // If there are no double-quotes around the string, then there are no escaped characters + // to decode, so just return + dubQuoteRegex := regexp.MustCompile(`^".+"$`) + if !dubQuoteRegex.MatchString(filename) { + return filename + } + // hack??/ + return string(UnescapeChars([]byte(filename))) + + // @todo special character support + // what we really need to do is to convert this into golang + // it seems that solution exists inside of "regexp" module + // either in "replaceAll" or in "doExecute" + // in the meantime, we do not support special characters in filenames or quotes + // // Need to hex encode '%' since we will be decoding the converted octal values from hex + // filename = filename.replace(/%/g, '%25'); + // // Replace all instances of octal literals with percent-encoded hex (ex. '\347\275\221' -> '%E7%BD%91'). + // // This is done because the octal literals represent UTF-8 bytes, and by converting them to percent-encoded + // // hex, we can use decodeURIComponent to get the Unicode chars. + // filename = filename.replace(/(?:\\(\d{1,3}))/g, (match, ...[octalValue, index, source]) => { + // // We need to make sure that the backslash is intended to escape the octal value. To do this, walk + // // backwards from the match to ensure that it's already escaped. + // const trailingBackslashes: RegExpMatchArray | null = (source as string) + // .slice(0, index as number) + // .match(/\\*$/); + // return trailingBackslashes && trailingBackslashes.length > 0 && trailingBackslashes[0].length % 2 === 0 + // ? `%${parseInt(octalValue, 8).toString(16)}` + // : match; + // }); + + // // Finally, decode the filename and unescape the escaped UTF-8 chars + // return JSON.parse(decodeURIComponent(filename)); + +} + +// gitStatus executes "git status" in a folder +func gitStatus(path string, gitPath string) (string, error) { + // log.Printf("[TRACE] gitStatus start") + // defer log.Printf("[TRACE] gitStatus end") + p := "git" + if len(gitPath) > 0 { + p = gitPath + } + cmd := exec.Command(p, "status", "-s", "-u", ".") + cmd.Dir = path + out, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("Failed to read git status: %w", err) + } + // log.Printf("[TRACE] gitStatus result: %v", strings.TrimSpace(string(out))) + return strings.TrimSpace(string(out)), nil +} + +func parseGitStatus(output string, PackagePath string) map[string]string { + // log.Printf("[TRACE] parseGitStatus start") + // defer log.Printf("[TRACE] parseGitStatus end") + changes := make(map[string]string) + + // Typically, output will look something like: + // M temp_modules/rush-package-deps-hash/package.json + // D package-deps-hash/src/index.ts + + // If there was an issue with `git ls-tree`, or there are no current changes, processOutputBlocks[1] + // will be empty or undefined + if len(output) == 0 { + // log.Printf("[TRACE] parseGitStatus result: no git changes") + return changes + } + // log.Printf("[TRACE] parseGitStatus result: found git changes") + gitRex := regexp.MustCompile(`("(\\"|[^"])+")|(\S+\s*)`) + // Note: The output of git hash-object uses \n newlines regardless of OS. + outputLines := strings.Split(output, "\n") + + for _, line := range outputLines { + if len(line) > 0 { + matches := gitRex.MatchString(line) + if matches == true { + // changeType is in the format of "XY" where "X" is the status of the file in the index and "Y" is the status of + // the file in the working tree. Some example statuses: + // - 'D' == deletion + // - 'M' == modification + // - 'A' == addition + // - '??' == untracked + // - 'R' == rename + // - 'RM' == rename with modifications + // - '[MARC]D' == deleted in work tree + // Full list of examples: https://git-scm.com/docs/git-status#_short_format + + // Lloks like this + //[["?? " "" "" "?? "] ["package_deps_hash_test.go" "" "" "package_deps_hash_test.go"]] + match := gitRex.FindAllStringSubmatch(line, -1) + if len(match[0]) > 1 { + changeType := match[0][0] + fileNameMatches := match[1][1:] + // log.Printf("match: %q", match) + // log.Printf("change: %v", strings.TrimRight(changeType, " ")) + + // We always care about the last filename in the filenames array. In the case of non-rename changes, + // the filenames array only contains one file, so we can join all segments that were split on spaces. + // In the case of rename changes, the last item in the array is the path to the file in the working tree, + // which is the only one that we care about. It is also surrounded by double-quotes if spaces are + // included, so no need to worry about joining different segments + lastFileName := strings.Join(fileNameMatches, "") + // looks like this + // [["R " "" "" "R "] ["turbo.config.js " "" "" "turbo.config.js "] ["-> " "" "" "-> "] ["turboooz.config.js" "" "" "turboooz.config.js"]] + if strings.HasPrefix(changeType, "R") { + lastFileName = strings.Join(match[len(match)-1][1:], "") + } + lastFileName = parseGitFilename(lastFileName) + // log.Printf(lastFileName) + changes[lastFileName] = strings.TrimRight(changeType, " ") + } + } + } + } + return changes +} diff --git a/cli/internal/globby/testdata/foo/package_deps_hash_test.go b/cli/internal/globby/testdata/foo/package_deps_hash_test.go new file mode 100644 index 0000000000000..be9e8ad39daa9 --- /dev/null +++ b/cli/internal/globby/testdata/foo/package_deps_hash_test.go @@ -0,0 +1,85 @@ +package fs + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_parseGitLsTree(t *testing.T) { + str := strings.TrimSpace(` + 100644 blob 7d10c39d8d500db5d7dc2040016a4678a1297f2e fs.go +100644 blob 96b98aca484a5f2775aa8fde07cfe5396a17693e hash.go +100644 blob b9fde9650a6f1cd86eab69e8442a85d89b1e0455 hash_test.go +100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/.test +100644 blob c7c5d4814cf152aa7b7b65f338bcb05d9d70402c test_data/test.txt +100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder++/test.txt +100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder1/a.txt +100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder1/sub_sub_folder/b.txt +100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder3/Zest.py +100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder3/best.py +100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder3/test.py +100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder4/TEST_BUILD +100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder4/test.py +100644 blob 8fd7339e6e8f7d203e61b7774fdef7692eb9c723 walk.go + `) + b1 := parseGitLsTree(str) + expected := map[string]string{ + "fs.go": "7d10c39d8d500db5d7dc2040016a4678a1297f2e", + "hash.go": "96b98aca484a5f2775aa8fde07cfe5396a17693e", + "hash_test.go": "b9fde9650a6f1cd86eab69e8442a85d89b1e0455", + "test_data/.test": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "test_data/test.txt": "c7c5d4814cf152aa7b7b65f338bcb05d9d70402c", + "test_data/test_subfolder++/test.txt": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "test_data/test_subfolder1/a.txt": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "test_data/test_subfolder1/sub_sub_folder/b.txt": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "test_data/test_subfolder3/Zest.py": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "test_data/test_subfolder3/best.py": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "test_data/test_subfolder3/test.py": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "test_data/test_subfolder4/TEST_BUILD": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "test_data/test_subfolder4/test.py": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "walk.go": "8fd7339e6e8f7d203e61b7774fdef7692eb9c723", + } + assert.EqualValues(t, expected, b1) +} + +// @todo special characters +// func Test_parseGitFilename(t *testing.T) { +// assert.EqualValues(t, `some/path/to/a/file name`, parseGitFilename(`some/path/to/a/file name`)) +// assert.EqualValues(t, `some/path/to/a/file name`, parseGitFilename(`some/path/to/a/file name`)) +// assert.EqualValues(t, `some/path/to/a/file?name`, parseGitFilename(`"some/path/to/a/file?name"`)) +// assert.EqualValues(t, `some/path/to/a/file\\name`, parseGitFilename(`"some/path/to/a/file\\\\name"`)) +// assert.EqualValues(t, `some/path/to/a/file"name`, parseGitFilename(`"some/path/to/a/file\\"name"`)) +// assert.EqualValues(t, `some/path/to/a/file"name`, parseGitFilename(`"some/path/to/a/file\\"name"`)) +// assert.EqualValues(t, `some/path/to/a/file网网name`, parseGitFilename(`"some/path/to/a/file\\347\\275\\221\\347\\275\\221name"`)) +// assert.EqualValues(t, `some/path/to/a/file\\347\\网name`, parseGitFilename(`"some/path/to/a/file\\\\347\\\\\\347\\275\\221name"`)) +// assert.EqualValues(t, `some/path/to/a/file\\网网name`, parseGitFilename(`"some/path/to/a/file\\\\\\347\\275\\221\\347\\275\\221name"`)) +// } + +func Test_parseGitStatus(t *testing.T) { + + want := map[string]string{ + "turboooz.config.js": "R", + "package_deps_hash.go": "??", + "package_deps_hash_test.go": "??", + } + input := ` +R turbo.config.js -> turboooz.config.js +?? package_deps_hash.go +?? package_deps_hash_test.go` + assert.EqualValues(t, want, parseGitStatus(input, "")) +} +func Test_getPackageDeps(t *testing.T) { + + want := map[string]string{ + "turboooz.config.js": "R", + "package_deps_hash.go": "??", + "package_deps_hash_test.go": "??", + } + input := ` +R turbo.config.js -> turboooz.config.js +?? package_deps_hash.go +?? package_deps_hash_test.go` + assert.EqualValues(t, want, parseGitStatus(input, "")) +} diff --git a/cli/internal/globby/testdata/foo/package_json.go b/cli/internal/globby/testdata/foo/package_json.go new file mode 100644 index 0000000000000..325a70d800135 --- /dev/null +++ b/cli/internal/globby/testdata/foo/package_json.go @@ -0,0 +1,115 @@ +package fs + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "reflect" + "sync" + + "github.com/pascaldekloe/name" +) + +// TurboCacheOptions are configuration for Turborepo cache + +type TurboConfigJSON struct { + Base string `json:"baseBranch,omitempty"` + GlobalDependencies []string `json:"globalDependencies,omitempty"` + TurboCacheOptions string `json:"cacheOptions,omitempty"` + Outputs []string `json:"outputs,omitempty"` + RemoteCacheUrl string `json:"remoteCacheUrl,omitempty"` + Pipeline map[string]Pipeline +} + +// Camelcase string with optional args. +func Camelcase(s string, v ...interface{}) string { + return name.CamelCase(fmt.Sprintf(s, v...), true) +} + +var requiredFields = []string{"Name", "Version"} + +type Pipeline struct { + Outputs []string `json:"outputs,omitempty"` + Cache *bool `json:"cache,omitempty"` + DependsOn []string `json:"dependsOn,omitempty"` +} + +// PackageJSON represents NodeJS package.json +type PackageJSON struct { + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` + Scripts map[string]string `json:"scripts,omitempty"` + Dependencies map[string]string `json:"dependencies,omitempty"` + DevDependencies map[string]string `json:"devDependencies,omitempty"` + OptionalDependencies map[string]string `json:"optionalDependencies,omitempty"` + PeerDependencies map[string]string `json:"peerDependencies,omitempty"` + Os []string `json:"os,omitempty"` + Workspaces Workspaces `json:"workspaces,omitempty"` + Private bool `json:"private,omitempty"` + PackageJSONPath string + Hash string + Dir string + InternalDeps []string + UnresolvedExternalDeps map[string]string + ExternalDeps []string + SubLockfile YarnLockfile + Turbo TurboConfigJSON `json:"turbo"` + Mu sync.Mutex + FilesHash string + ExternalDepsHash string +} + +type Workspaces []string + +type WorkspacesAlt struct { + Packages []string `json:"packages,omitempty"` +} + +func (r *Workspaces) UnmarshalJSON(data []byte) error { + var tmp = &WorkspacesAlt{} + if err := json.Unmarshal(data, &tmp); err == nil { + *r = Workspaces(tmp.Packages) + return nil + } + var tempstr = []string{} + if err := json.Unmarshal(data, &tempstr); err != nil { + return err + } + *r = tempstr + return nil +} + +// Parse parses package.json payload and returns structure. +func Parse(payload []byte) (*PackageJSON, error) { + var packagejson *PackageJSON + err := json.Unmarshal(payload, &packagejson) + return packagejson, err +} + +// Validate checks if provided package.json is valid. +func (p *PackageJSON) Validate() error { + for _, fieldname := range requiredFields { + value := getField(p, fieldname) + if len(value) == 0 { + return fmt.Errorf("'%s' field is required in package.json", fieldname) + } + } + + return nil +} + +// getField returns struct field value by name. +func getField(i interface{}, fieldname string) string { + value := reflect.ValueOf(i) + field := reflect.Indirect(value).FieldByName(fieldname) + return field.String() +} + +// ReadPackageJSON returns a struct of package.json +func ReadPackageJSON(path string) (*PackageJSON, error) { + b, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + return Parse(b) +} diff --git a/cli/internal/globby/testdata/package_deps_hash.go b/cli/internal/globby/testdata/package_deps_hash.go new file mode 100644 index 0000000000000..23af9681b2d16 --- /dev/null +++ b/cli/internal/globby/testdata/package_deps_hash.go @@ -0,0 +1,290 @@ +package fs + +import ( + "bytes" + "fmt" + "os/exec" + "path/filepath" + "regexp" + "strings" + "turbo/internal/util" +) + +// Predefine []byte variables to avoid runtime allocations. +var ( + escapedSlash = []byte(`\\`) + regularSlash = []byte(`\`) + escapedTab = []byte(`\t`) + regularTab = []byte("\t") +) + +// PackageDepsOptions are parameters for getting git hashes for a filesystem +type PackageDepsOptions struct { + // PackagePath is the folder path to derive the package dependencies from. This is typically the folder + // containing package.json. If omitted, the default value is the current working directory. + PackagePath string + // ExcludedPaths is an optional array of file path exclusions. If a file should be omitted from the list + // of dependencies, use this to exclude it. + ExcludedPaths []string + // GitPath is an optional alternative path to the git installation + GitPath string +} + +// GetPackageDeps Builds an object containing git hashes for the files under the specified `packagePath` folder. +func GetPackageDeps(p *PackageDepsOptions) (map[string]string, error) { + gitLsOutput, err := gitLsTree(p.PackagePath, p.GitPath) + if err != nil { + return nil, fmt.Errorf("Could not get git hashes for files in package %s: %w", p.PackagePath, err) + } + // Add all the checked in hashes. + result := parseGitLsTree(gitLsOutput) + + if len(p.ExcludedPaths) > 0 { + for _, p := range p.ExcludedPaths { + // @todo explore optimization + delete(result, p) + } + } + + // Update the checked in hashes with the current repo status + gitStatusOutput, err := gitStatus(p.PackagePath, p.GitPath) + if err != nil { + return nil, err + } + currentlyChangedFiles := parseGitStatus(gitStatusOutput, p.PackagePath) + var filesToHash []string + excludedPathsSet := new(util.Set) + for filename, changeType := range currentlyChangedFiles { + if changeType == "D" || (len(changeType) == 2 && string(changeType)[1] == []byte("D")[0]) { + delete(result, filename) + } else { + if !excludedPathsSet.Include(filename) { + filesToHash = append(filesToHash, filename) + } + } + } + + // log.Printf("[TRACE] %v:", gitStatusOutput) + // log.Printf("[TRACE] start GitHashForFiles") + current, err := GitHashForFiles( + filesToHash, + p.PackagePath, + ) + if err != nil { + return nil, fmt.Errorf("could not retrieve git hash for files in %s", p.PackagePath) + } + // log.Printf("[TRACE] end GitHashForFiles") + // log.Printf("[TRACE] GitHashForFiles files %v", current) + for filename, hash := range current { + // log.Printf("[TRACE] GitHashForFiles files %v: %v", filename, hash) + result[filename] = hash + } + // log.Printf("[TRACE] GitHashForFiles result %v", result) + return result, nil +} + +// GitHashForFiles a list of files returns a map of with their git hash values. It uses +// git hash-object under the +func GitHashForFiles(filesToHash []string, PackagePath string) (map[string]string, error) { + changes := make(map[string]string) + if len(filesToHash) > 0 { + var input = []string{"hash-object"} + + for _, filename := range filesToHash { + input = append(input, filepath.Join(PackagePath, filename)) + } + // fmt.Println(input) + cmd := exec.Command("git", input...) + // https://blog.kowalczyk.info/article/wOYk/advanced-command-execution-in-go-with-osexec.html + cmd.Stdin = strings.NewReader(strings.Join(input, "\n")) + cmd.Dir = PackagePath + out, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("git hash-object exited with status: %w", err) + } + offByOne := strings.Split(string(out), "\n") // there is an extra "" + hashes := offByOne[:len(offByOne)-1] + if len(hashes) != len(filesToHash) { + return nil, fmt.Errorf("passed %v file paths to Git to hash, but received %v hashes.", len(filesToHash), len(hashes)) + } + for i, hash := range hashes { + filepath := filesToHash[i] + changes[filepath] = hash + } + } + + return changes, nil +} + +// UnescapeChars reverses escaped characters. +func UnescapeChars(in []byte) []byte { + if bytes.ContainsAny(in, "\\\t") { + return in + } + + out := bytes.Replace(in, escapedSlash, regularSlash, -1) + out = bytes.Replace(out, escapedTab, regularTab, -1) + return out +} + +// gitLsTree executes "git ls-tree" in a folder +func gitLsTree(path string, gitPath string) (string, error) { + + cmd := exec.Command("git", "ls-tree", "HEAD", "-r") + cmd.Dir = path + out, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("Failed to read `git ls-tree`: %w", err) + } + return strings.TrimSpace(string(out)), nil +} + +func parseGitLsTree(output string) map[string]string { + changes := make(map[string]string) + if len(output) > 0 { + // A line is expected to look like: + // 100644 blob 3451bccdc831cb43d7a70ed8e628dcf9c7f888c8 src/typings/tsd.d.ts + // 160000 commit c5880bf5b0c6c1f2e2c43c95beeb8f0a808e8bac rushstack + gitRex := regexp.MustCompile(`([0-9]{6})\s(blob|commit)\s([a-f0-9]{40})\s*(.*)`) + outputLines := strings.Split(output, "\n") + + for _, line := range outputLines { + if len(line) > 0 { + matches := gitRex.MatchString(line) + if matches == true { + // this looks like this + // [["160000 commit c5880bf5b0c6c1f2e2c43c95beeb8f0a808e8bac rushstack" "160000" "commit" "c5880bf5b0c6c1f2e2c43c95beeb8f0a808e8bac" "rushstack"]] + match := gitRex.FindAllStringSubmatch(line, -1) + if len(match[0][3]) > 0 && len(match[0][4]) > 0 { + hash := match[0][3] + filename := parseGitFilename(match[0][4]) + changes[filename] = hash + } + // @todo error + } + } + } + } + return changes +} + +// Couldn't figure out how to deal with special characters. Skipping for now. +// @todo see https://github.com/microsoft/rushstack/blob/925ad8c9e22997c1edf5fe38c53fa618e8180f70/libraries/package-deps-hash/src/getPackageDeps.ts#L19 +func parseGitFilename(filename string) string { + // If there are no double-quotes around the string, then there are no escaped characters + // to decode, so just return + dubQuoteRegex := regexp.MustCompile(`^".+"$`) + if !dubQuoteRegex.MatchString(filename) { + return filename + } + // hack??/ + return string(UnescapeChars([]byte(filename))) + + // @todo special character support + // what we really need to do is to convert this into golang + // it seems that solution exists inside of "regexp" module + // either in "replaceAll" or in "doExecute" + // in the meantime, we do not support special characters in filenames or quotes + // // Need to hex encode '%' since we will be decoding the converted octal values from hex + // filename = filename.replace(/%/g, '%25'); + // // Replace all instances of octal literals with percent-encoded hex (ex. '\347\275\221' -> '%E7%BD%91'). + // // This is done because the octal literals represent UTF-8 bytes, and by converting them to percent-encoded + // // hex, we can use decodeURIComponent to get the Unicode chars. + // filename = filename.replace(/(?:\\(\d{1,3}))/g, (match, ...[octalValue, index, source]) => { + // // We need to make sure that the backslash is intended to escape the octal value. To do this, walk + // // backwards from the match to ensure that it's already escaped. + // const trailingBackslashes: RegExpMatchArray | null = (source as string) + // .slice(0, index as number) + // .match(/\\*$/); + // return trailingBackslashes && trailingBackslashes.length > 0 && trailingBackslashes[0].length % 2 === 0 + // ? `%${parseInt(octalValue, 8).toString(16)}` + // : match; + // }); + + // // Finally, decode the filename and unescape the escaped UTF-8 chars + // return JSON.parse(decodeURIComponent(filename)); + +} + +// gitStatus executes "git status" in a folder +func gitStatus(path string, gitPath string) (string, error) { + // log.Printf("[TRACE] gitStatus start") + // defer log.Printf("[TRACE] gitStatus end") + p := "git" + if len(gitPath) > 0 { + p = gitPath + } + cmd := exec.Command(p, "status", "-s", "-u", ".") + cmd.Dir = path + out, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("Failed to read git status: %w", err) + } + // log.Printf("[TRACE] gitStatus result: %v", strings.TrimSpace(string(out))) + return strings.TrimSpace(string(out)), nil +} + +func parseGitStatus(output string, PackagePath string) map[string]string { + // log.Printf("[TRACE] parseGitStatus start") + // defer log.Printf("[TRACE] parseGitStatus end") + changes := make(map[string]string) + + // Typically, output will look something like: + // M temp_modules/rush-package-deps-hash/package.json + // D package-deps-hash/src/index.ts + + // If there was an issue with `git ls-tree`, or there are no current changes, processOutputBlocks[1] + // will be empty or undefined + if len(output) == 0 { + // log.Printf("[TRACE] parseGitStatus result: no git changes") + return changes + } + // log.Printf("[TRACE] parseGitStatus result: found git changes") + gitRex := regexp.MustCompile(`("(\\"|[^"])+")|(\S+\s*)`) + // Note: The output of git hash-object uses \n newlines regardless of OS. + outputLines := strings.Split(output, "\n") + + for _, line := range outputLines { + if len(line) > 0 { + matches := gitRex.MatchString(line) + if matches == true { + // changeType is in the format of "XY" where "X" is the status of the file in the index and "Y" is the status of + // the file in the working tree. Some example statuses: + // - 'D' == deletion + // - 'M' == modification + // - 'A' == addition + // - '??' == untracked + // - 'R' == rename + // - 'RM' == rename with modifications + // - '[MARC]D' == deleted in work tree + // Full list of examples: https://git-scm.com/docs/git-status#_short_format + + // Lloks like this + //[["?? " "" "" "?? "] ["package_deps_hash_test.go" "" "" "package_deps_hash_test.go"]] + match := gitRex.FindAllStringSubmatch(line, -1) + if len(match[0]) > 1 { + changeType := match[0][0] + fileNameMatches := match[1][1:] + // log.Printf("match: %q", match) + // log.Printf("change: %v", strings.TrimRight(changeType, " ")) + + // We always care about the last filename in the filenames array. In the case of non-rename changes, + // the filenames array only contains one file, so we can join all segments that were split on spaces. + // In the case of rename changes, the last item in the array is the path to the file in the working tree, + // which is the only one that we care about. It is also surrounded by double-quotes if spaces are + // included, so no need to worry about joining different segments + lastFileName := strings.Join(fileNameMatches, "") + // looks like this + // [["R " "" "" "R "] ["turbo.config.js " "" "" "turbo.config.js "] ["-> " "" "" "-> "] ["turboooz.config.js" "" "" "turboooz.config.js"]] + if strings.HasPrefix(changeType, "R") { + lastFileName = strings.Join(match[len(match)-1][1:], "") + } + lastFileName = parseGitFilename(lastFileName) + // log.Printf(lastFileName) + changes[lastFileName] = strings.TrimRight(changeType, " ") + } + } + } + } + return changes +} diff --git a/cli/internal/globby/testdata/package_deps_hash_test.go b/cli/internal/globby/testdata/package_deps_hash_test.go new file mode 100644 index 0000000000000..be9e8ad39daa9 --- /dev/null +++ b/cli/internal/globby/testdata/package_deps_hash_test.go @@ -0,0 +1,85 @@ +package fs + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_parseGitLsTree(t *testing.T) { + str := strings.TrimSpace(` + 100644 blob 7d10c39d8d500db5d7dc2040016a4678a1297f2e fs.go +100644 blob 96b98aca484a5f2775aa8fde07cfe5396a17693e hash.go +100644 blob b9fde9650a6f1cd86eab69e8442a85d89b1e0455 hash_test.go +100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/.test +100644 blob c7c5d4814cf152aa7b7b65f338bcb05d9d70402c test_data/test.txt +100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder++/test.txt +100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder1/a.txt +100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder1/sub_sub_folder/b.txt +100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder3/Zest.py +100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder3/best.py +100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder3/test.py +100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder4/TEST_BUILD +100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder4/test.py +100644 blob 8fd7339e6e8f7d203e61b7774fdef7692eb9c723 walk.go + `) + b1 := parseGitLsTree(str) + expected := map[string]string{ + "fs.go": "7d10c39d8d500db5d7dc2040016a4678a1297f2e", + "hash.go": "96b98aca484a5f2775aa8fde07cfe5396a17693e", + "hash_test.go": "b9fde9650a6f1cd86eab69e8442a85d89b1e0455", + "test_data/.test": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "test_data/test.txt": "c7c5d4814cf152aa7b7b65f338bcb05d9d70402c", + "test_data/test_subfolder++/test.txt": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "test_data/test_subfolder1/a.txt": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "test_data/test_subfolder1/sub_sub_folder/b.txt": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "test_data/test_subfolder3/Zest.py": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "test_data/test_subfolder3/best.py": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "test_data/test_subfolder3/test.py": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "test_data/test_subfolder4/TEST_BUILD": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "test_data/test_subfolder4/test.py": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", + "walk.go": "8fd7339e6e8f7d203e61b7774fdef7692eb9c723", + } + assert.EqualValues(t, expected, b1) +} + +// @todo special characters +// func Test_parseGitFilename(t *testing.T) { +// assert.EqualValues(t, `some/path/to/a/file name`, parseGitFilename(`some/path/to/a/file name`)) +// assert.EqualValues(t, `some/path/to/a/file name`, parseGitFilename(`some/path/to/a/file name`)) +// assert.EqualValues(t, `some/path/to/a/file?name`, parseGitFilename(`"some/path/to/a/file?name"`)) +// assert.EqualValues(t, `some/path/to/a/file\\name`, parseGitFilename(`"some/path/to/a/file\\\\name"`)) +// assert.EqualValues(t, `some/path/to/a/file"name`, parseGitFilename(`"some/path/to/a/file\\"name"`)) +// assert.EqualValues(t, `some/path/to/a/file"name`, parseGitFilename(`"some/path/to/a/file\\"name"`)) +// assert.EqualValues(t, `some/path/to/a/file网网name`, parseGitFilename(`"some/path/to/a/file\\347\\275\\221\\347\\275\\221name"`)) +// assert.EqualValues(t, `some/path/to/a/file\\347\\网name`, parseGitFilename(`"some/path/to/a/file\\\\347\\\\\\347\\275\\221name"`)) +// assert.EqualValues(t, `some/path/to/a/file\\网网name`, parseGitFilename(`"some/path/to/a/file\\\\\\347\\275\\221\\347\\275\\221name"`)) +// } + +func Test_parseGitStatus(t *testing.T) { + + want := map[string]string{ + "turboooz.config.js": "R", + "package_deps_hash.go": "??", + "package_deps_hash_test.go": "??", + } + input := ` +R turbo.config.js -> turboooz.config.js +?? package_deps_hash.go +?? package_deps_hash_test.go` + assert.EqualValues(t, want, parseGitStatus(input, "")) +} +func Test_getPackageDeps(t *testing.T) { + + want := map[string]string{ + "turboooz.config.js": "R", + "package_deps_hash.go": "??", + "package_deps_hash_test.go": "??", + } + input := ` +R turbo.config.js -> turboooz.config.js +?? package_deps_hash.go +?? package_deps_hash_test.go` + assert.EqualValues(t, want, parseGitStatus(input, "")) +} diff --git a/cli/internal/globby/testdata/package_json.go b/cli/internal/globby/testdata/package_json.go new file mode 100644 index 0000000000000..325a70d800135 --- /dev/null +++ b/cli/internal/globby/testdata/package_json.go @@ -0,0 +1,115 @@ +package fs + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "reflect" + "sync" + + "github.com/pascaldekloe/name" +) + +// TurboCacheOptions are configuration for Turborepo cache + +type TurboConfigJSON struct { + Base string `json:"baseBranch,omitempty"` + GlobalDependencies []string `json:"globalDependencies,omitempty"` + TurboCacheOptions string `json:"cacheOptions,omitempty"` + Outputs []string `json:"outputs,omitempty"` + RemoteCacheUrl string `json:"remoteCacheUrl,omitempty"` + Pipeline map[string]Pipeline +} + +// Camelcase string with optional args. +func Camelcase(s string, v ...interface{}) string { + return name.CamelCase(fmt.Sprintf(s, v...), true) +} + +var requiredFields = []string{"Name", "Version"} + +type Pipeline struct { + Outputs []string `json:"outputs,omitempty"` + Cache *bool `json:"cache,omitempty"` + DependsOn []string `json:"dependsOn,omitempty"` +} + +// PackageJSON represents NodeJS package.json +type PackageJSON struct { + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` + Scripts map[string]string `json:"scripts,omitempty"` + Dependencies map[string]string `json:"dependencies,omitempty"` + DevDependencies map[string]string `json:"devDependencies,omitempty"` + OptionalDependencies map[string]string `json:"optionalDependencies,omitempty"` + PeerDependencies map[string]string `json:"peerDependencies,omitempty"` + Os []string `json:"os,omitempty"` + Workspaces Workspaces `json:"workspaces,omitempty"` + Private bool `json:"private,omitempty"` + PackageJSONPath string + Hash string + Dir string + InternalDeps []string + UnresolvedExternalDeps map[string]string + ExternalDeps []string + SubLockfile YarnLockfile + Turbo TurboConfigJSON `json:"turbo"` + Mu sync.Mutex + FilesHash string + ExternalDepsHash string +} + +type Workspaces []string + +type WorkspacesAlt struct { + Packages []string `json:"packages,omitempty"` +} + +func (r *Workspaces) UnmarshalJSON(data []byte) error { + var tmp = &WorkspacesAlt{} + if err := json.Unmarshal(data, &tmp); err == nil { + *r = Workspaces(tmp.Packages) + return nil + } + var tempstr = []string{} + if err := json.Unmarshal(data, &tempstr); err != nil { + return err + } + *r = tempstr + return nil +} + +// Parse parses package.json payload and returns structure. +func Parse(payload []byte) (*PackageJSON, error) { + var packagejson *PackageJSON + err := json.Unmarshal(payload, &packagejson) + return packagejson, err +} + +// Validate checks if provided package.json is valid. +func (p *PackageJSON) Validate() error { + for _, fieldname := range requiredFields { + value := getField(p, fieldname) + if len(value) == 0 { + return fmt.Errorf("'%s' field is required in package.json", fieldname) + } + } + + return nil +} + +// getField returns struct field value by name. +func getField(i interface{}, fieldname string) string { + value := reflect.ValueOf(i) + field := reflect.Indirect(value).FieldByName(fieldname) + return field.String() +} + +// ReadPackageJSON returns a struct of package.json +func ReadPackageJSON(path string) (*PackageJSON, error) { + b, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + return Parse(b) +} From 3889311c958d60c93eef4cf7f9e66df7a78caa3d Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Thu, 16 Dec 2021 23:46:16 -0500 Subject: [PATCH 3/5] Fix glob callsites --- cli/internal/context/context.go | 4 ++-- cli/internal/run/run.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index 32d65d8082f0b..913a2f0bc7e4e 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -180,7 +180,7 @@ func WithGraph(rootpath string, config *config.Config) Option { globalDeps := make(util.Set) if len(pkg.Turbo.GlobalDependencies) > 0 { - f := globby.GlobFiles(rootpath, &pkg.Turbo.GlobalDependencies, nil) + f := globby.GlobFiles(rootpath, pkg.Turbo.GlobalDependencies, []string{}) for _, val := range f { globalDeps.Add(val) } @@ -243,7 +243,7 @@ func WithGraph(rootpath string, config *config.Config) Option { "**/tests/**/*", } - f := globby.GlobFiles(rootpath, &justJsons, &ignore) + f := globby.GlobFiles(rootpath, justJsons, ignore) for i, val := range f { _, val := i, val // https://golang.org/doc/faq#closures_and_goroutines diff --git a/cli/internal/run/run.go b/cli/internal/run/run.go index 4ae8a93f0adc6..44a8370962cb5 100644 --- a/cli/internal/run/run.go +++ b/cli/internal/run/run.go @@ -583,7 +583,7 @@ func (c *RunCommand) Run(args []string) int { if runOptions.cache && (pipeline.Cache == nil || *pipeline.Cache) { targetLogger.Debug("caching output", "outputs", outputs) ignore := []string{} - filesToBeCached := globby.GlobFiles(pack.Dir, &outputs, &ignore) + filesToBeCached := globby.GlobFiles(pack.Dir, outputs, ignore) if err := turboCache.Put(pack.Dir, hash, int(time.Since(cmdTime).Milliseconds()), filesToBeCached); err != nil { c.logError(targetLogger, "", fmt.Errorf("Error caching output: %w", err)) } From 7ddf6fd6c235c434e06cd228c6274a9d4eb9a04f Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Thu, 16 Dec 2021 23:48:13 -0500 Subject: [PATCH 4/5] Clean up globby port --- cli/internal/globby/globby.go | 16 +- cli/internal/globby/globby_test.go | 40 --- .../globby/testdata/foo/package_deps_hash.go | 290 ------------------ .../testdata/foo/package_deps_hash_test.go | 85 ----- .../globby/testdata/foo/package_json.go | 115 ------- .../globby/testdata/package_deps_hash.go | 290 ------------------ .../globby/testdata/package_deps_hash_test.go | 85 ----- cli/internal/globby/testdata/package_json.go | 115 ------- 8 files changed, 8 insertions(+), 1028 deletions(-) delete mode 100644 cli/internal/globby/globby_test.go delete mode 100644 cli/internal/globby/testdata/foo/package_deps_hash.go delete mode 100644 cli/internal/globby/testdata/foo/package_deps_hash_test.go delete mode 100644 cli/internal/globby/testdata/foo/package_json.go delete mode 100644 cli/internal/globby/testdata/package_deps_hash.go delete mode 100644 cli/internal/globby/testdata/package_deps_hash_test.go delete mode 100644 cli/internal/globby/testdata/package_json.go diff --git a/cli/internal/globby/globby.go b/cli/internal/globby/globby.go index be92752d03362..bbc0ac1087789 100644 --- a/cli/internal/globby/globby.go +++ b/cli/internal/globby/globby.go @@ -9,23 +9,23 @@ import ( "github.com/bmatcuk/doublestar/v4" ) -func GlobFiles(ws_path string, include_pattens []string, exclude_pattens []string) []string { +func GlobFiles(ws_path string, includePatterns []string, excludePatterns []string) []string { var include []string var exclude []string var result []string - for _, p := range include_pattens { + for _, p := range includePatterns { include = append(include, filepath.Join(ws_path, p)) } - for _, p := range exclude_pattens { + for _, p := range excludePatterns { exclude = append(exclude, filepath.Join(ws_path, p)) } - var include_pattern = "{" + strings.Join(include, ",") + "}" - var exclude_pattern = "{" + strings.Join(exclude, ",") + "}" - var _ = fs.Walk(ws_path, func(p string, isDir bool) error { - if val, _ := doublestar.PathMatch(exclude_pattern, p); val { + includePattern := "{" + strings.Join(include, ",") + "}" + excludePattern := "{" + strings.Join(exclude, ",") + "}" + _ = fs.Walk(ws_path, func(p string, isDir bool) error { + if val, _ := doublestar.PathMatch(excludePattern, p); val { if isDir { return filepath.SkipDir } @@ -36,7 +36,7 @@ func GlobFiles(ws_path string, include_pattens []string, exclude_pattens []strin return nil } - if val, _ := doublestar.PathMatch(include_pattern, p); val || len(include_pattens) == 0 { + if val, _ := doublestar.PathMatch(includePattern, p); val || len(includePatterns) == 0 { result = append(result, p) } diff --git a/cli/internal/globby/globby_test.go b/cli/internal/globby/globby_test.go deleted file mode 100644 index 87adc34bc4e53..0000000000000 --- a/cli/internal/globby/globby_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package globby - -import ( - "reflect" - "testing" -) - -func TestGlobFiles(t *testing.T) { - - type args struct { - ws_path string - include_pattens []string - exclude_pattens []string - } - tests := []struct { - name string - args args - want []string - }{ - { - name: "globFiles", - args: args{ - ws_path: "testdata", - include_pattens: []string{"package/**/*.go"}, - exclude_pattens: []string{"**/node_modules/**"}, - }, - want: []string{ - "package_deps_hash.go", - }, - }, - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := GlobFiles(tt.args.ws_path, tt.args.include_pattens, tt.args.exclude_pattens); !reflect.DeepEqual(got, tt.want) { - t.Errorf("GlobFiles() = %#v, want %#v", got, tt.want) - } - }) - } -} diff --git a/cli/internal/globby/testdata/foo/package_deps_hash.go b/cli/internal/globby/testdata/foo/package_deps_hash.go deleted file mode 100644 index 23af9681b2d16..0000000000000 --- a/cli/internal/globby/testdata/foo/package_deps_hash.go +++ /dev/null @@ -1,290 +0,0 @@ -package fs - -import ( - "bytes" - "fmt" - "os/exec" - "path/filepath" - "regexp" - "strings" - "turbo/internal/util" -) - -// Predefine []byte variables to avoid runtime allocations. -var ( - escapedSlash = []byte(`\\`) - regularSlash = []byte(`\`) - escapedTab = []byte(`\t`) - regularTab = []byte("\t") -) - -// PackageDepsOptions are parameters for getting git hashes for a filesystem -type PackageDepsOptions struct { - // PackagePath is the folder path to derive the package dependencies from. This is typically the folder - // containing package.json. If omitted, the default value is the current working directory. - PackagePath string - // ExcludedPaths is an optional array of file path exclusions. If a file should be omitted from the list - // of dependencies, use this to exclude it. - ExcludedPaths []string - // GitPath is an optional alternative path to the git installation - GitPath string -} - -// GetPackageDeps Builds an object containing git hashes for the files under the specified `packagePath` folder. -func GetPackageDeps(p *PackageDepsOptions) (map[string]string, error) { - gitLsOutput, err := gitLsTree(p.PackagePath, p.GitPath) - if err != nil { - return nil, fmt.Errorf("Could not get git hashes for files in package %s: %w", p.PackagePath, err) - } - // Add all the checked in hashes. - result := parseGitLsTree(gitLsOutput) - - if len(p.ExcludedPaths) > 0 { - for _, p := range p.ExcludedPaths { - // @todo explore optimization - delete(result, p) - } - } - - // Update the checked in hashes with the current repo status - gitStatusOutput, err := gitStatus(p.PackagePath, p.GitPath) - if err != nil { - return nil, err - } - currentlyChangedFiles := parseGitStatus(gitStatusOutput, p.PackagePath) - var filesToHash []string - excludedPathsSet := new(util.Set) - for filename, changeType := range currentlyChangedFiles { - if changeType == "D" || (len(changeType) == 2 && string(changeType)[1] == []byte("D")[0]) { - delete(result, filename) - } else { - if !excludedPathsSet.Include(filename) { - filesToHash = append(filesToHash, filename) - } - } - } - - // log.Printf("[TRACE] %v:", gitStatusOutput) - // log.Printf("[TRACE] start GitHashForFiles") - current, err := GitHashForFiles( - filesToHash, - p.PackagePath, - ) - if err != nil { - return nil, fmt.Errorf("could not retrieve git hash for files in %s", p.PackagePath) - } - // log.Printf("[TRACE] end GitHashForFiles") - // log.Printf("[TRACE] GitHashForFiles files %v", current) - for filename, hash := range current { - // log.Printf("[TRACE] GitHashForFiles files %v: %v", filename, hash) - result[filename] = hash - } - // log.Printf("[TRACE] GitHashForFiles result %v", result) - return result, nil -} - -// GitHashForFiles a list of files returns a map of with their git hash values. It uses -// git hash-object under the -func GitHashForFiles(filesToHash []string, PackagePath string) (map[string]string, error) { - changes := make(map[string]string) - if len(filesToHash) > 0 { - var input = []string{"hash-object"} - - for _, filename := range filesToHash { - input = append(input, filepath.Join(PackagePath, filename)) - } - // fmt.Println(input) - cmd := exec.Command("git", input...) - // https://blog.kowalczyk.info/article/wOYk/advanced-command-execution-in-go-with-osexec.html - cmd.Stdin = strings.NewReader(strings.Join(input, "\n")) - cmd.Dir = PackagePath - out, err := cmd.CombinedOutput() - if err != nil { - return nil, fmt.Errorf("git hash-object exited with status: %w", err) - } - offByOne := strings.Split(string(out), "\n") // there is an extra "" - hashes := offByOne[:len(offByOne)-1] - if len(hashes) != len(filesToHash) { - return nil, fmt.Errorf("passed %v file paths to Git to hash, but received %v hashes.", len(filesToHash), len(hashes)) - } - for i, hash := range hashes { - filepath := filesToHash[i] - changes[filepath] = hash - } - } - - return changes, nil -} - -// UnescapeChars reverses escaped characters. -func UnescapeChars(in []byte) []byte { - if bytes.ContainsAny(in, "\\\t") { - return in - } - - out := bytes.Replace(in, escapedSlash, regularSlash, -1) - out = bytes.Replace(out, escapedTab, regularTab, -1) - return out -} - -// gitLsTree executes "git ls-tree" in a folder -func gitLsTree(path string, gitPath string) (string, error) { - - cmd := exec.Command("git", "ls-tree", "HEAD", "-r") - cmd.Dir = path - out, err := cmd.CombinedOutput() - if err != nil { - return "", fmt.Errorf("Failed to read `git ls-tree`: %w", err) - } - return strings.TrimSpace(string(out)), nil -} - -func parseGitLsTree(output string) map[string]string { - changes := make(map[string]string) - if len(output) > 0 { - // A line is expected to look like: - // 100644 blob 3451bccdc831cb43d7a70ed8e628dcf9c7f888c8 src/typings/tsd.d.ts - // 160000 commit c5880bf5b0c6c1f2e2c43c95beeb8f0a808e8bac rushstack - gitRex := regexp.MustCompile(`([0-9]{6})\s(blob|commit)\s([a-f0-9]{40})\s*(.*)`) - outputLines := strings.Split(output, "\n") - - for _, line := range outputLines { - if len(line) > 0 { - matches := gitRex.MatchString(line) - if matches == true { - // this looks like this - // [["160000 commit c5880bf5b0c6c1f2e2c43c95beeb8f0a808e8bac rushstack" "160000" "commit" "c5880bf5b0c6c1f2e2c43c95beeb8f0a808e8bac" "rushstack"]] - match := gitRex.FindAllStringSubmatch(line, -1) - if len(match[0][3]) > 0 && len(match[0][4]) > 0 { - hash := match[0][3] - filename := parseGitFilename(match[0][4]) - changes[filename] = hash - } - // @todo error - } - } - } - } - return changes -} - -// Couldn't figure out how to deal with special characters. Skipping for now. -// @todo see https://github.com/microsoft/rushstack/blob/925ad8c9e22997c1edf5fe38c53fa618e8180f70/libraries/package-deps-hash/src/getPackageDeps.ts#L19 -func parseGitFilename(filename string) string { - // If there are no double-quotes around the string, then there are no escaped characters - // to decode, so just return - dubQuoteRegex := regexp.MustCompile(`^".+"$`) - if !dubQuoteRegex.MatchString(filename) { - return filename - } - // hack??/ - return string(UnescapeChars([]byte(filename))) - - // @todo special character support - // what we really need to do is to convert this into golang - // it seems that solution exists inside of "regexp" module - // either in "replaceAll" or in "doExecute" - // in the meantime, we do not support special characters in filenames or quotes - // // Need to hex encode '%' since we will be decoding the converted octal values from hex - // filename = filename.replace(/%/g, '%25'); - // // Replace all instances of octal literals with percent-encoded hex (ex. '\347\275\221' -> '%E7%BD%91'). - // // This is done because the octal literals represent UTF-8 bytes, and by converting them to percent-encoded - // // hex, we can use decodeURIComponent to get the Unicode chars. - // filename = filename.replace(/(?:\\(\d{1,3}))/g, (match, ...[octalValue, index, source]) => { - // // We need to make sure that the backslash is intended to escape the octal value. To do this, walk - // // backwards from the match to ensure that it's already escaped. - // const trailingBackslashes: RegExpMatchArray | null = (source as string) - // .slice(0, index as number) - // .match(/\\*$/); - // return trailingBackslashes && trailingBackslashes.length > 0 && trailingBackslashes[0].length % 2 === 0 - // ? `%${parseInt(octalValue, 8).toString(16)}` - // : match; - // }); - - // // Finally, decode the filename and unescape the escaped UTF-8 chars - // return JSON.parse(decodeURIComponent(filename)); - -} - -// gitStatus executes "git status" in a folder -func gitStatus(path string, gitPath string) (string, error) { - // log.Printf("[TRACE] gitStatus start") - // defer log.Printf("[TRACE] gitStatus end") - p := "git" - if len(gitPath) > 0 { - p = gitPath - } - cmd := exec.Command(p, "status", "-s", "-u", ".") - cmd.Dir = path - out, err := cmd.CombinedOutput() - if err != nil { - return "", fmt.Errorf("Failed to read git status: %w", err) - } - // log.Printf("[TRACE] gitStatus result: %v", strings.TrimSpace(string(out))) - return strings.TrimSpace(string(out)), nil -} - -func parseGitStatus(output string, PackagePath string) map[string]string { - // log.Printf("[TRACE] parseGitStatus start") - // defer log.Printf("[TRACE] parseGitStatus end") - changes := make(map[string]string) - - // Typically, output will look something like: - // M temp_modules/rush-package-deps-hash/package.json - // D package-deps-hash/src/index.ts - - // If there was an issue with `git ls-tree`, or there are no current changes, processOutputBlocks[1] - // will be empty or undefined - if len(output) == 0 { - // log.Printf("[TRACE] parseGitStatus result: no git changes") - return changes - } - // log.Printf("[TRACE] parseGitStatus result: found git changes") - gitRex := regexp.MustCompile(`("(\\"|[^"])+")|(\S+\s*)`) - // Note: The output of git hash-object uses \n newlines regardless of OS. - outputLines := strings.Split(output, "\n") - - for _, line := range outputLines { - if len(line) > 0 { - matches := gitRex.MatchString(line) - if matches == true { - // changeType is in the format of "XY" where "X" is the status of the file in the index and "Y" is the status of - // the file in the working tree. Some example statuses: - // - 'D' == deletion - // - 'M' == modification - // - 'A' == addition - // - '??' == untracked - // - 'R' == rename - // - 'RM' == rename with modifications - // - '[MARC]D' == deleted in work tree - // Full list of examples: https://git-scm.com/docs/git-status#_short_format - - // Lloks like this - //[["?? " "" "" "?? "] ["package_deps_hash_test.go" "" "" "package_deps_hash_test.go"]] - match := gitRex.FindAllStringSubmatch(line, -1) - if len(match[0]) > 1 { - changeType := match[0][0] - fileNameMatches := match[1][1:] - // log.Printf("match: %q", match) - // log.Printf("change: %v", strings.TrimRight(changeType, " ")) - - // We always care about the last filename in the filenames array. In the case of non-rename changes, - // the filenames array only contains one file, so we can join all segments that were split on spaces. - // In the case of rename changes, the last item in the array is the path to the file in the working tree, - // which is the only one that we care about. It is also surrounded by double-quotes if spaces are - // included, so no need to worry about joining different segments - lastFileName := strings.Join(fileNameMatches, "") - // looks like this - // [["R " "" "" "R "] ["turbo.config.js " "" "" "turbo.config.js "] ["-> " "" "" "-> "] ["turboooz.config.js" "" "" "turboooz.config.js"]] - if strings.HasPrefix(changeType, "R") { - lastFileName = strings.Join(match[len(match)-1][1:], "") - } - lastFileName = parseGitFilename(lastFileName) - // log.Printf(lastFileName) - changes[lastFileName] = strings.TrimRight(changeType, " ") - } - } - } - } - return changes -} diff --git a/cli/internal/globby/testdata/foo/package_deps_hash_test.go b/cli/internal/globby/testdata/foo/package_deps_hash_test.go deleted file mode 100644 index be9e8ad39daa9..0000000000000 --- a/cli/internal/globby/testdata/foo/package_deps_hash_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package fs - -import ( - "strings" - "testing" - - "github.com/stretchr/testify/assert" -) - -func Test_parseGitLsTree(t *testing.T) { - str := strings.TrimSpace(` - 100644 blob 7d10c39d8d500db5d7dc2040016a4678a1297f2e fs.go -100644 blob 96b98aca484a5f2775aa8fde07cfe5396a17693e hash.go -100644 blob b9fde9650a6f1cd86eab69e8442a85d89b1e0455 hash_test.go -100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/.test -100644 blob c7c5d4814cf152aa7b7b65f338bcb05d9d70402c test_data/test.txt -100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder++/test.txt -100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder1/a.txt -100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder1/sub_sub_folder/b.txt -100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder3/Zest.py -100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder3/best.py -100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder3/test.py -100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder4/TEST_BUILD -100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder4/test.py -100644 blob 8fd7339e6e8f7d203e61b7774fdef7692eb9c723 walk.go - `) - b1 := parseGitLsTree(str) - expected := map[string]string{ - "fs.go": "7d10c39d8d500db5d7dc2040016a4678a1297f2e", - "hash.go": "96b98aca484a5f2775aa8fde07cfe5396a17693e", - "hash_test.go": "b9fde9650a6f1cd86eab69e8442a85d89b1e0455", - "test_data/.test": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", - "test_data/test.txt": "c7c5d4814cf152aa7b7b65f338bcb05d9d70402c", - "test_data/test_subfolder++/test.txt": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", - "test_data/test_subfolder1/a.txt": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", - "test_data/test_subfolder1/sub_sub_folder/b.txt": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", - "test_data/test_subfolder3/Zest.py": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", - "test_data/test_subfolder3/best.py": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", - "test_data/test_subfolder3/test.py": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", - "test_data/test_subfolder4/TEST_BUILD": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", - "test_data/test_subfolder4/test.py": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", - "walk.go": "8fd7339e6e8f7d203e61b7774fdef7692eb9c723", - } - assert.EqualValues(t, expected, b1) -} - -// @todo special characters -// func Test_parseGitFilename(t *testing.T) { -// assert.EqualValues(t, `some/path/to/a/file name`, parseGitFilename(`some/path/to/a/file name`)) -// assert.EqualValues(t, `some/path/to/a/file name`, parseGitFilename(`some/path/to/a/file name`)) -// assert.EqualValues(t, `some/path/to/a/file?name`, parseGitFilename(`"some/path/to/a/file?name"`)) -// assert.EqualValues(t, `some/path/to/a/file\\name`, parseGitFilename(`"some/path/to/a/file\\\\name"`)) -// assert.EqualValues(t, `some/path/to/a/file"name`, parseGitFilename(`"some/path/to/a/file\\"name"`)) -// assert.EqualValues(t, `some/path/to/a/file"name`, parseGitFilename(`"some/path/to/a/file\\"name"`)) -// assert.EqualValues(t, `some/path/to/a/file网网name`, parseGitFilename(`"some/path/to/a/file\\347\\275\\221\\347\\275\\221name"`)) -// assert.EqualValues(t, `some/path/to/a/file\\347\\网name`, parseGitFilename(`"some/path/to/a/file\\\\347\\\\\\347\\275\\221name"`)) -// assert.EqualValues(t, `some/path/to/a/file\\网网name`, parseGitFilename(`"some/path/to/a/file\\\\\\347\\275\\221\\347\\275\\221name"`)) -// } - -func Test_parseGitStatus(t *testing.T) { - - want := map[string]string{ - "turboooz.config.js": "R", - "package_deps_hash.go": "??", - "package_deps_hash_test.go": "??", - } - input := ` -R turbo.config.js -> turboooz.config.js -?? package_deps_hash.go -?? package_deps_hash_test.go` - assert.EqualValues(t, want, parseGitStatus(input, "")) -} -func Test_getPackageDeps(t *testing.T) { - - want := map[string]string{ - "turboooz.config.js": "R", - "package_deps_hash.go": "??", - "package_deps_hash_test.go": "??", - } - input := ` -R turbo.config.js -> turboooz.config.js -?? package_deps_hash.go -?? package_deps_hash_test.go` - assert.EqualValues(t, want, parseGitStatus(input, "")) -} diff --git a/cli/internal/globby/testdata/foo/package_json.go b/cli/internal/globby/testdata/foo/package_json.go deleted file mode 100644 index 325a70d800135..0000000000000 --- a/cli/internal/globby/testdata/foo/package_json.go +++ /dev/null @@ -1,115 +0,0 @@ -package fs - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "reflect" - "sync" - - "github.com/pascaldekloe/name" -) - -// TurboCacheOptions are configuration for Turborepo cache - -type TurboConfigJSON struct { - Base string `json:"baseBranch,omitempty"` - GlobalDependencies []string `json:"globalDependencies,omitempty"` - TurboCacheOptions string `json:"cacheOptions,omitempty"` - Outputs []string `json:"outputs,omitempty"` - RemoteCacheUrl string `json:"remoteCacheUrl,omitempty"` - Pipeline map[string]Pipeline -} - -// Camelcase string with optional args. -func Camelcase(s string, v ...interface{}) string { - return name.CamelCase(fmt.Sprintf(s, v...), true) -} - -var requiredFields = []string{"Name", "Version"} - -type Pipeline struct { - Outputs []string `json:"outputs,omitempty"` - Cache *bool `json:"cache,omitempty"` - DependsOn []string `json:"dependsOn,omitempty"` -} - -// PackageJSON represents NodeJS package.json -type PackageJSON struct { - Name string `json:"name,omitempty"` - Version string `json:"version,omitempty"` - Scripts map[string]string `json:"scripts,omitempty"` - Dependencies map[string]string `json:"dependencies,omitempty"` - DevDependencies map[string]string `json:"devDependencies,omitempty"` - OptionalDependencies map[string]string `json:"optionalDependencies,omitempty"` - PeerDependencies map[string]string `json:"peerDependencies,omitempty"` - Os []string `json:"os,omitempty"` - Workspaces Workspaces `json:"workspaces,omitempty"` - Private bool `json:"private,omitempty"` - PackageJSONPath string - Hash string - Dir string - InternalDeps []string - UnresolvedExternalDeps map[string]string - ExternalDeps []string - SubLockfile YarnLockfile - Turbo TurboConfigJSON `json:"turbo"` - Mu sync.Mutex - FilesHash string - ExternalDepsHash string -} - -type Workspaces []string - -type WorkspacesAlt struct { - Packages []string `json:"packages,omitempty"` -} - -func (r *Workspaces) UnmarshalJSON(data []byte) error { - var tmp = &WorkspacesAlt{} - if err := json.Unmarshal(data, &tmp); err == nil { - *r = Workspaces(tmp.Packages) - return nil - } - var tempstr = []string{} - if err := json.Unmarshal(data, &tempstr); err != nil { - return err - } - *r = tempstr - return nil -} - -// Parse parses package.json payload and returns structure. -func Parse(payload []byte) (*PackageJSON, error) { - var packagejson *PackageJSON - err := json.Unmarshal(payload, &packagejson) - return packagejson, err -} - -// Validate checks if provided package.json is valid. -func (p *PackageJSON) Validate() error { - for _, fieldname := range requiredFields { - value := getField(p, fieldname) - if len(value) == 0 { - return fmt.Errorf("'%s' field is required in package.json", fieldname) - } - } - - return nil -} - -// getField returns struct field value by name. -func getField(i interface{}, fieldname string) string { - value := reflect.ValueOf(i) - field := reflect.Indirect(value).FieldByName(fieldname) - return field.String() -} - -// ReadPackageJSON returns a struct of package.json -func ReadPackageJSON(path string) (*PackageJSON, error) { - b, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - return Parse(b) -} diff --git a/cli/internal/globby/testdata/package_deps_hash.go b/cli/internal/globby/testdata/package_deps_hash.go deleted file mode 100644 index 23af9681b2d16..0000000000000 --- a/cli/internal/globby/testdata/package_deps_hash.go +++ /dev/null @@ -1,290 +0,0 @@ -package fs - -import ( - "bytes" - "fmt" - "os/exec" - "path/filepath" - "regexp" - "strings" - "turbo/internal/util" -) - -// Predefine []byte variables to avoid runtime allocations. -var ( - escapedSlash = []byte(`\\`) - regularSlash = []byte(`\`) - escapedTab = []byte(`\t`) - regularTab = []byte("\t") -) - -// PackageDepsOptions are parameters for getting git hashes for a filesystem -type PackageDepsOptions struct { - // PackagePath is the folder path to derive the package dependencies from. This is typically the folder - // containing package.json. If omitted, the default value is the current working directory. - PackagePath string - // ExcludedPaths is an optional array of file path exclusions. If a file should be omitted from the list - // of dependencies, use this to exclude it. - ExcludedPaths []string - // GitPath is an optional alternative path to the git installation - GitPath string -} - -// GetPackageDeps Builds an object containing git hashes for the files under the specified `packagePath` folder. -func GetPackageDeps(p *PackageDepsOptions) (map[string]string, error) { - gitLsOutput, err := gitLsTree(p.PackagePath, p.GitPath) - if err != nil { - return nil, fmt.Errorf("Could not get git hashes for files in package %s: %w", p.PackagePath, err) - } - // Add all the checked in hashes. - result := parseGitLsTree(gitLsOutput) - - if len(p.ExcludedPaths) > 0 { - for _, p := range p.ExcludedPaths { - // @todo explore optimization - delete(result, p) - } - } - - // Update the checked in hashes with the current repo status - gitStatusOutput, err := gitStatus(p.PackagePath, p.GitPath) - if err != nil { - return nil, err - } - currentlyChangedFiles := parseGitStatus(gitStatusOutput, p.PackagePath) - var filesToHash []string - excludedPathsSet := new(util.Set) - for filename, changeType := range currentlyChangedFiles { - if changeType == "D" || (len(changeType) == 2 && string(changeType)[1] == []byte("D")[0]) { - delete(result, filename) - } else { - if !excludedPathsSet.Include(filename) { - filesToHash = append(filesToHash, filename) - } - } - } - - // log.Printf("[TRACE] %v:", gitStatusOutput) - // log.Printf("[TRACE] start GitHashForFiles") - current, err := GitHashForFiles( - filesToHash, - p.PackagePath, - ) - if err != nil { - return nil, fmt.Errorf("could not retrieve git hash for files in %s", p.PackagePath) - } - // log.Printf("[TRACE] end GitHashForFiles") - // log.Printf("[TRACE] GitHashForFiles files %v", current) - for filename, hash := range current { - // log.Printf("[TRACE] GitHashForFiles files %v: %v", filename, hash) - result[filename] = hash - } - // log.Printf("[TRACE] GitHashForFiles result %v", result) - return result, nil -} - -// GitHashForFiles a list of files returns a map of with their git hash values. It uses -// git hash-object under the -func GitHashForFiles(filesToHash []string, PackagePath string) (map[string]string, error) { - changes := make(map[string]string) - if len(filesToHash) > 0 { - var input = []string{"hash-object"} - - for _, filename := range filesToHash { - input = append(input, filepath.Join(PackagePath, filename)) - } - // fmt.Println(input) - cmd := exec.Command("git", input...) - // https://blog.kowalczyk.info/article/wOYk/advanced-command-execution-in-go-with-osexec.html - cmd.Stdin = strings.NewReader(strings.Join(input, "\n")) - cmd.Dir = PackagePath - out, err := cmd.CombinedOutput() - if err != nil { - return nil, fmt.Errorf("git hash-object exited with status: %w", err) - } - offByOne := strings.Split(string(out), "\n") // there is an extra "" - hashes := offByOne[:len(offByOne)-1] - if len(hashes) != len(filesToHash) { - return nil, fmt.Errorf("passed %v file paths to Git to hash, but received %v hashes.", len(filesToHash), len(hashes)) - } - for i, hash := range hashes { - filepath := filesToHash[i] - changes[filepath] = hash - } - } - - return changes, nil -} - -// UnescapeChars reverses escaped characters. -func UnescapeChars(in []byte) []byte { - if bytes.ContainsAny(in, "\\\t") { - return in - } - - out := bytes.Replace(in, escapedSlash, regularSlash, -1) - out = bytes.Replace(out, escapedTab, regularTab, -1) - return out -} - -// gitLsTree executes "git ls-tree" in a folder -func gitLsTree(path string, gitPath string) (string, error) { - - cmd := exec.Command("git", "ls-tree", "HEAD", "-r") - cmd.Dir = path - out, err := cmd.CombinedOutput() - if err != nil { - return "", fmt.Errorf("Failed to read `git ls-tree`: %w", err) - } - return strings.TrimSpace(string(out)), nil -} - -func parseGitLsTree(output string) map[string]string { - changes := make(map[string]string) - if len(output) > 0 { - // A line is expected to look like: - // 100644 blob 3451bccdc831cb43d7a70ed8e628dcf9c7f888c8 src/typings/tsd.d.ts - // 160000 commit c5880bf5b0c6c1f2e2c43c95beeb8f0a808e8bac rushstack - gitRex := regexp.MustCompile(`([0-9]{6})\s(blob|commit)\s([a-f0-9]{40})\s*(.*)`) - outputLines := strings.Split(output, "\n") - - for _, line := range outputLines { - if len(line) > 0 { - matches := gitRex.MatchString(line) - if matches == true { - // this looks like this - // [["160000 commit c5880bf5b0c6c1f2e2c43c95beeb8f0a808e8bac rushstack" "160000" "commit" "c5880bf5b0c6c1f2e2c43c95beeb8f0a808e8bac" "rushstack"]] - match := gitRex.FindAllStringSubmatch(line, -1) - if len(match[0][3]) > 0 && len(match[0][4]) > 0 { - hash := match[0][3] - filename := parseGitFilename(match[0][4]) - changes[filename] = hash - } - // @todo error - } - } - } - } - return changes -} - -// Couldn't figure out how to deal with special characters. Skipping for now. -// @todo see https://github.com/microsoft/rushstack/blob/925ad8c9e22997c1edf5fe38c53fa618e8180f70/libraries/package-deps-hash/src/getPackageDeps.ts#L19 -func parseGitFilename(filename string) string { - // If there are no double-quotes around the string, then there are no escaped characters - // to decode, so just return - dubQuoteRegex := regexp.MustCompile(`^".+"$`) - if !dubQuoteRegex.MatchString(filename) { - return filename - } - // hack??/ - return string(UnescapeChars([]byte(filename))) - - // @todo special character support - // what we really need to do is to convert this into golang - // it seems that solution exists inside of "regexp" module - // either in "replaceAll" or in "doExecute" - // in the meantime, we do not support special characters in filenames or quotes - // // Need to hex encode '%' since we will be decoding the converted octal values from hex - // filename = filename.replace(/%/g, '%25'); - // // Replace all instances of octal literals with percent-encoded hex (ex. '\347\275\221' -> '%E7%BD%91'). - // // This is done because the octal literals represent UTF-8 bytes, and by converting them to percent-encoded - // // hex, we can use decodeURIComponent to get the Unicode chars. - // filename = filename.replace(/(?:\\(\d{1,3}))/g, (match, ...[octalValue, index, source]) => { - // // We need to make sure that the backslash is intended to escape the octal value. To do this, walk - // // backwards from the match to ensure that it's already escaped. - // const trailingBackslashes: RegExpMatchArray | null = (source as string) - // .slice(0, index as number) - // .match(/\\*$/); - // return trailingBackslashes && trailingBackslashes.length > 0 && trailingBackslashes[0].length % 2 === 0 - // ? `%${parseInt(octalValue, 8).toString(16)}` - // : match; - // }); - - // // Finally, decode the filename and unescape the escaped UTF-8 chars - // return JSON.parse(decodeURIComponent(filename)); - -} - -// gitStatus executes "git status" in a folder -func gitStatus(path string, gitPath string) (string, error) { - // log.Printf("[TRACE] gitStatus start") - // defer log.Printf("[TRACE] gitStatus end") - p := "git" - if len(gitPath) > 0 { - p = gitPath - } - cmd := exec.Command(p, "status", "-s", "-u", ".") - cmd.Dir = path - out, err := cmd.CombinedOutput() - if err != nil { - return "", fmt.Errorf("Failed to read git status: %w", err) - } - // log.Printf("[TRACE] gitStatus result: %v", strings.TrimSpace(string(out))) - return strings.TrimSpace(string(out)), nil -} - -func parseGitStatus(output string, PackagePath string) map[string]string { - // log.Printf("[TRACE] parseGitStatus start") - // defer log.Printf("[TRACE] parseGitStatus end") - changes := make(map[string]string) - - // Typically, output will look something like: - // M temp_modules/rush-package-deps-hash/package.json - // D package-deps-hash/src/index.ts - - // If there was an issue with `git ls-tree`, or there are no current changes, processOutputBlocks[1] - // will be empty or undefined - if len(output) == 0 { - // log.Printf("[TRACE] parseGitStatus result: no git changes") - return changes - } - // log.Printf("[TRACE] parseGitStatus result: found git changes") - gitRex := regexp.MustCompile(`("(\\"|[^"])+")|(\S+\s*)`) - // Note: The output of git hash-object uses \n newlines regardless of OS. - outputLines := strings.Split(output, "\n") - - for _, line := range outputLines { - if len(line) > 0 { - matches := gitRex.MatchString(line) - if matches == true { - // changeType is in the format of "XY" where "X" is the status of the file in the index and "Y" is the status of - // the file in the working tree. Some example statuses: - // - 'D' == deletion - // - 'M' == modification - // - 'A' == addition - // - '??' == untracked - // - 'R' == rename - // - 'RM' == rename with modifications - // - '[MARC]D' == deleted in work tree - // Full list of examples: https://git-scm.com/docs/git-status#_short_format - - // Lloks like this - //[["?? " "" "" "?? "] ["package_deps_hash_test.go" "" "" "package_deps_hash_test.go"]] - match := gitRex.FindAllStringSubmatch(line, -1) - if len(match[0]) > 1 { - changeType := match[0][0] - fileNameMatches := match[1][1:] - // log.Printf("match: %q", match) - // log.Printf("change: %v", strings.TrimRight(changeType, " ")) - - // We always care about the last filename in the filenames array. In the case of non-rename changes, - // the filenames array only contains one file, so we can join all segments that were split on spaces. - // In the case of rename changes, the last item in the array is the path to the file in the working tree, - // which is the only one that we care about. It is also surrounded by double-quotes if spaces are - // included, so no need to worry about joining different segments - lastFileName := strings.Join(fileNameMatches, "") - // looks like this - // [["R " "" "" "R "] ["turbo.config.js " "" "" "turbo.config.js "] ["-> " "" "" "-> "] ["turboooz.config.js" "" "" "turboooz.config.js"]] - if strings.HasPrefix(changeType, "R") { - lastFileName = strings.Join(match[len(match)-1][1:], "") - } - lastFileName = parseGitFilename(lastFileName) - // log.Printf(lastFileName) - changes[lastFileName] = strings.TrimRight(changeType, " ") - } - } - } - } - return changes -} diff --git a/cli/internal/globby/testdata/package_deps_hash_test.go b/cli/internal/globby/testdata/package_deps_hash_test.go deleted file mode 100644 index be9e8ad39daa9..0000000000000 --- a/cli/internal/globby/testdata/package_deps_hash_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package fs - -import ( - "strings" - "testing" - - "github.com/stretchr/testify/assert" -) - -func Test_parseGitLsTree(t *testing.T) { - str := strings.TrimSpace(` - 100644 blob 7d10c39d8d500db5d7dc2040016a4678a1297f2e fs.go -100644 blob 96b98aca484a5f2775aa8fde07cfe5396a17693e hash.go -100644 blob b9fde9650a6f1cd86eab69e8442a85d89b1e0455 hash_test.go -100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/.test -100644 blob c7c5d4814cf152aa7b7b65f338bcb05d9d70402c test_data/test.txt -100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder++/test.txt -100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder1/a.txt -100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder1/sub_sub_folder/b.txt -100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder3/Zest.py -100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder3/best.py -100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder3/test.py -100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder4/TEST_BUILD -100644 blob e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 test_data/test_subfolder4/test.py -100644 blob 8fd7339e6e8f7d203e61b7774fdef7692eb9c723 walk.go - `) - b1 := parseGitLsTree(str) - expected := map[string]string{ - "fs.go": "7d10c39d8d500db5d7dc2040016a4678a1297f2e", - "hash.go": "96b98aca484a5f2775aa8fde07cfe5396a17693e", - "hash_test.go": "b9fde9650a6f1cd86eab69e8442a85d89b1e0455", - "test_data/.test": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", - "test_data/test.txt": "c7c5d4814cf152aa7b7b65f338bcb05d9d70402c", - "test_data/test_subfolder++/test.txt": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", - "test_data/test_subfolder1/a.txt": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", - "test_data/test_subfolder1/sub_sub_folder/b.txt": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", - "test_data/test_subfolder3/Zest.py": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", - "test_data/test_subfolder3/best.py": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", - "test_data/test_subfolder3/test.py": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", - "test_data/test_subfolder4/TEST_BUILD": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", - "test_data/test_subfolder4/test.py": "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", - "walk.go": "8fd7339e6e8f7d203e61b7774fdef7692eb9c723", - } - assert.EqualValues(t, expected, b1) -} - -// @todo special characters -// func Test_parseGitFilename(t *testing.T) { -// assert.EqualValues(t, `some/path/to/a/file name`, parseGitFilename(`some/path/to/a/file name`)) -// assert.EqualValues(t, `some/path/to/a/file name`, parseGitFilename(`some/path/to/a/file name`)) -// assert.EqualValues(t, `some/path/to/a/file?name`, parseGitFilename(`"some/path/to/a/file?name"`)) -// assert.EqualValues(t, `some/path/to/a/file\\name`, parseGitFilename(`"some/path/to/a/file\\\\name"`)) -// assert.EqualValues(t, `some/path/to/a/file"name`, parseGitFilename(`"some/path/to/a/file\\"name"`)) -// assert.EqualValues(t, `some/path/to/a/file"name`, parseGitFilename(`"some/path/to/a/file\\"name"`)) -// assert.EqualValues(t, `some/path/to/a/file网网name`, parseGitFilename(`"some/path/to/a/file\\347\\275\\221\\347\\275\\221name"`)) -// assert.EqualValues(t, `some/path/to/a/file\\347\\网name`, parseGitFilename(`"some/path/to/a/file\\\\347\\\\\\347\\275\\221name"`)) -// assert.EqualValues(t, `some/path/to/a/file\\网网name`, parseGitFilename(`"some/path/to/a/file\\\\\\347\\275\\221\\347\\275\\221name"`)) -// } - -func Test_parseGitStatus(t *testing.T) { - - want := map[string]string{ - "turboooz.config.js": "R", - "package_deps_hash.go": "??", - "package_deps_hash_test.go": "??", - } - input := ` -R turbo.config.js -> turboooz.config.js -?? package_deps_hash.go -?? package_deps_hash_test.go` - assert.EqualValues(t, want, parseGitStatus(input, "")) -} -func Test_getPackageDeps(t *testing.T) { - - want := map[string]string{ - "turboooz.config.js": "R", - "package_deps_hash.go": "??", - "package_deps_hash_test.go": "??", - } - input := ` -R turbo.config.js -> turboooz.config.js -?? package_deps_hash.go -?? package_deps_hash_test.go` - assert.EqualValues(t, want, parseGitStatus(input, "")) -} diff --git a/cli/internal/globby/testdata/package_json.go b/cli/internal/globby/testdata/package_json.go deleted file mode 100644 index 325a70d800135..0000000000000 --- a/cli/internal/globby/testdata/package_json.go +++ /dev/null @@ -1,115 +0,0 @@ -package fs - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "reflect" - "sync" - - "github.com/pascaldekloe/name" -) - -// TurboCacheOptions are configuration for Turborepo cache - -type TurboConfigJSON struct { - Base string `json:"baseBranch,omitempty"` - GlobalDependencies []string `json:"globalDependencies,omitempty"` - TurboCacheOptions string `json:"cacheOptions,omitempty"` - Outputs []string `json:"outputs,omitempty"` - RemoteCacheUrl string `json:"remoteCacheUrl,omitempty"` - Pipeline map[string]Pipeline -} - -// Camelcase string with optional args. -func Camelcase(s string, v ...interface{}) string { - return name.CamelCase(fmt.Sprintf(s, v...), true) -} - -var requiredFields = []string{"Name", "Version"} - -type Pipeline struct { - Outputs []string `json:"outputs,omitempty"` - Cache *bool `json:"cache,omitempty"` - DependsOn []string `json:"dependsOn,omitempty"` -} - -// PackageJSON represents NodeJS package.json -type PackageJSON struct { - Name string `json:"name,omitempty"` - Version string `json:"version,omitempty"` - Scripts map[string]string `json:"scripts,omitempty"` - Dependencies map[string]string `json:"dependencies,omitempty"` - DevDependencies map[string]string `json:"devDependencies,omitempty"` - OptionalDependencies map[string]string `json:"optionalDependencies,omitempty"` - PeerDependencies map[string]string `json:"peerDependencies,omitempty"` - Os []string `json:"os,omitempty"` - Workspaces Workspaces `json:"workspaces,omitempty"` - Private bool `json:"private,omitempty"` - PackageJSONPath string - Hash string - Dir string - InternalDeps []string - UnresolvedExternalDeps map[string]string - ExternalDeps []string - SubLockfile YarnLockfile - Turbo TurboConfigJSON `json:"turbo"` - Mu sync.Mutex - FilesHash string - ExternalDepsHash string -} - -type Workspaces []string - -type WorkspacesAlt struct { - Packages []string `json:"packages,omitempty"` -} - -func (r *Workspaces) UnmarshalJSON(data []byte) error { - var tmp = &WorkspacesAlt{} - if err := json.Unmarshal(data, &tmp); err == nil { - *r = Workspaces(tmp.Packages) - return nil - } - var tempstr = []string{} - if err := json.Unmarshal(data, &tempstr); err != nil { - return err - } - *r = tempstr - return nil -} - -// Parse parses package.json payload and returns structure. -func Parse(payload []byte) (*PackageJSON, error) { - var packagejson *PackageJSON - err := json.Unmarshal(payload, &packagejson) - return packagejson, err -} - -// Validate checks if provided package.json is valid. -func (p *PackageJSON) Validate() error { - for _, fieldname := range requiredFields { - value := getField(p, fieldname) - if len(value) == 0 { - return fmt.Errorf("'%s' field is required in package.json", fieldname) - } - } - - return nil -} - -// getField returns struct field value by name. -func getField(i interface{}, fieldname string) string { - value := reflect.ValueOf(i) - field := reflect.Indirect(value).FieldByName(fieldname) - return field.String() -} - -// ReadPackageJSON returns a struct of package.json -func ReadPackageJSON(path string) (*PackageJSON, error) { - b, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - return Parse(b) -} From b0f4473a9ba418d531753cfe02ef4e0a1bc3d628 Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Thu, 16 Dec 2021 23:48:39 -0500 Subject: [PATCH 5/5] Rename to baseDir --- cli/internal/globby/globby.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/internal/globby/globby.go b/cli/internal/globby/globby.go index bbc0ac1087789..66174f9857454 100644 --- a/cli/internal/globby/globby.go +++ b/cli/internal/globby/globby.go @@ -9,22 +9,22 @@ import ( "github.com/bmatcuk/doublestar/v4" ) -func GlobFiles(ws_path string, includePatterns []string, excludePatterns []string) []string { +func GlobFiles(basePath string, includePatterns []string, excludePatterns []string) []string { var include []string var exclude []string var result []string for _, p := range includePatterns { - include = append(include, filepath.Join(ws_path, p)) + include = append(include, filepath.Join(basePath, p)) } for _, p := range excludePatterns { - exclude = append(exclude, filepath.Join(ws_path, p)) + exclude = append(exclude, filepath.Join(basePath, p)) } includePattern := "{" + strings.Join(include, ",") + "}" excludePattern := "{" + strings.Join(exclude, ",") + "}" - _ = fs.Walk(ws_path, func(p string, isDir bool) error { + _ = fs.Walk(basePath, func(p string, isDir bool) error { if val, _ := doublestar.PathMatch(excludePattern, p); val { if isDir { return filepath.SkipDir