From 7d36cd171d678b235bfd1a1b707a4abe62a77b8c Mon Sep 17 00:00:00 2001 From: Etourneau Gwenn Date: Thu, 19 Jun 2025 15:18:16 +0900 Subject: [PATCH 1/3] Add Module Json --- go.mod | 4 ++- go.sum | 11 +++--- src/data.go | 18 ++++++++-- src/data_test.go | 3 +- src/modules.go | 87 ++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 113 insertions(+), 10 deletions(-) create mode 100644 src/modules.go diff --git a/go.mod b/go.mod index 5d5baedf..c71c1e7d 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/google/go-github/v47 v47.1.0 github.com/hashicorp/go-version v1.7.0 github.com/hashicorp/hc-install v0.9.2 - github.com/hashicorp/hcl/v2 v2.23.0 + github.com/hashicorp/hcl/v2 v2.23.1-0.20250203194505-ba0759438da2 github.com/hashicorp/terraform-exec v0.23.0 github.com/jameswoolfenden/identity v0.0.5 github.com/rs/zerolog v1.34.0 @@ -52,7 +52,9 @@ require ( github.com/cyphar/filepath-securejoin v0.4.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect + github.com/fatih/color v1.18.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.6.2 // indirect github.com/go-logr/logr v1.4.3 // indirect diff --git a/go.sum b/go.sum index 6bde059c..169fadf7 100644 --- a/go.sum +++ b/go.sum @@ -65,13 +65,14 @@ github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= -github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= -github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= @@ -130,8 +131,8 @@ github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKe github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/hc-install v0.9.2 h1:v80EtNX4fCVHqzL9Lg/2xkp62bbvQMnvPQ0G+OmtO24= github.com/hashicorp/hc-install v0.9.2/go.mod h1:XUqBQNnuT4RsxoxiM9ZaUk0NX8hi2h+Lb6/c0OZnC/I= -github.com/hashicorp/hcl/v2 v2.23.0 h1:Fphj1/gCylPxHutVSEOf2fBOh1VE4AuLV7+kbJf3qos= -github.com/hashicorp/hcl/v2 v2.23.0/go.mod h1:62ZYHrXgPoX8xBnzl8QzbWq4dyDsDtfCRgIq1rbJEvA= +github.com/hashicorp/hcl/v2 v2.23.1-0.20250203194505-ba0759438da2 h1:JP8y98OtHTujECs4s/HxlKc5yql/RlC99Dt1Iz4R+lM= +github.com/hashicorp/hcl/v2 v2.23.1-0.20250203194505-ba0759438da2/go.mod h1:k+HgkLpoWu9OS81sy4j1XKDXaWm/rLysG33v5ibdDnc= github.com/hashicorp/terraform-exec v0.23.0 h1:MUiBM1s0CNlRFsCLJuM5wXZrzA3MnPYEsiXmzATMW/I= github.com/hashicorp/terraform-exec v0.23.0/go.mod h1:mA+qnx1R8eePycfwKkCRk3Wy65mwInvlpAeOwmA7vlY= github.com/hashicorp/terraform-json v0.25.0 h1:rmNqc/CIfcWawGiwXmRuiXJKEiJu1ntGoxseG1hLhoQ= diff --git a/src/data.go b/src/data.go index b3f6876e..1d359b41 100644 --- a/src/data.go +++ b/src/data.go @@ -37,6 +37,11 @@ func GetResources(file string, dirName string) ([]ResourceV2, error) { return Resources, err } + moduleJson, err := GetModuleJson(filepath.Join(dirName, "/", ".terraform", "modules")) + if err != nil { + return Resources, err + } + for _, block := range temp.Blocks { var resource ResourceV2 resource.TypeName = block.Type @@ -50,7 +55,7 @@ func GetResources(file string, dirName string) ([]ResourceV2, error) { } case module: { - LocalResources, err := GetLocalModules(block, dirName) + LocalResources, err := GetLocalModules(block, dirName, moduleJson) if err == nil { Resources = append(LocalResources, Resources...) } else { @@ -147,7 +152,7 @@ func GetResourceBlocks(file string) (*hclsyntax.Body, error) { } // GetLocalModules return resource from a path. -func GetLocalModules(block *hclsyntax.Block, dirName string) ([]ResourceV2, error) { +func GetLocalModules(block *hclsyntax.Block, dirName string, listModulesJson ModuleJson) ([]ResourceV2, error) { var Resources []ResourceV2 modulePath := GetModulePath(block) @@ -157,9 +162,12 @@ func GetLocalModules(block *hclsyntax.Block, dirName string) ([]ResourceV2, erro return nil, &gitReferenceError{modulePath} } + // Module is coming from HCP Terraform Cloud + if strings.HasPrefix(modulePath, "app.terraform.io") { + modulePath = ReturnLocalAddrFromSource(modulePath, listModulesJson) + } // have the path to the module modulePath = filepath.Join(dirName, "/", modulePath) - // now process these extras ExtraFiles, err := GetTF(modulePath) if err != nil { @@ -273,3 +281,7 @@ func GetPermission(result ResourceV2) (Sorted, error) { return myPermission, err } + +func GetModuleJson(dir string) (ModuleJson, error) { + return ReadModuleJsonForDir(dir) +} diff --git a/src/data_test.go b/src/data_test.go index 74d694e3..96a7f926 100644 --- a/src/data_test.go +++ b/src/data_test.go @@ -107,6 +107,7 @@ func Test_getLocalModules(t *testing.T) { notLocal, _ := filepath.Abs("../testdata/modules/examples/notlocal") notBlock := getInitialBlock(notLocal + "/module.local.tf") + moduleJson := make(pike.ModuleJson) tests := []struct { name string args args @@ -132,7 +133,7 @@ func Test_getLocalModules(t *testing.T) { for _, tt := range tests { // t.Parallel() t.Run(tt.name, func(t *testing.T) { - got, err := pike.GetLocalModules(tt.args.block, tt.args.dirName) + got, err := pike.GetLocalModules(tt.args.block, tt.args.dirName, moduleJson) if (err != nil) != tt.wantErr { t.Errorf("GetResources() error = %v, wantErr %v", err, tt.wantErr) diff --git a/src/modules.go b/src/modules.go new file mode 100644 index 00000000..4103767a --- /dev/null +++ b/src/modules.go @@ -0,0 +1,87 @@ +package pike + +import ( + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + + version "github.com/hashicorp/go-version" +) + +const ( + ManifestSnapshotFilename = "modules.json" +) + +// Record represents some metadata about an installed module, as part +// of a modules json. +type Record struct { + Key string `json:"Key"` + SourceAddr string `json:"Source"` + Version *version.Version `json:"-"` + VersionStr string `json:"Version,omitempty"` + Dir string `json:"Dir"` +} + +type ModuleJson map[string]Record + +type modulesJson struct { + Records []Record `json:"Modules"` +} + +func ReadModuleJson(r io.Reader) (ModuleJson, error) { + src, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + if len(src) == 0 { + return make(ModuleJson), nil + } + + var read modulesJson + err = json.Unmarshal(src, &read) + if err != nil { + return nil, fmt.Errorf("error unmarshalling snapshot: %v", err) + } + new := make(ModuleJson) + for _, record := range read.Records { + if record.VersionStr != "" { + record.Version, err = version.NewVersion(record.VersionStr) + if err != nil { + return nil, fmt.Errorf("invalid version %q for %s: %s", record.VersionStr, record.Key, err) + } + } + // Ensure Windows is using the proper modules path format after + // reading the modules manifest Dir records + record.Dir = filepath.FromSlash(record.Dir) + + if _, exists := new[record.Key]; exists { + return nil, fmt.Errorf("snapshot file contains two records for path %s", record.Key) + } + new[record.Key] = record + } + return new, nil +} + +func ReadModuleJsonForDir(dir string) (ModuleJson, error) { + fn := filepath.Join(dir, ManifestSnapshotFilename) + r, err := os.Open(fn) + if err != nil { + if os.IsNotExist(err) { + return make(ModuleJson), nil + } + return nil, err + } + return ReadModuleJson(r) +} + +func ReturnLocalAddrFromSource(source string, listModules ModuleJson) string { + for _, module := range listModules { + if module.SourceAddr == source { + return module.Dir + } + } + return "" +} From 11cd6a319860c977b547d012fda50ce51b1046e8 Mon Sep 17 00:00:00 2001 From: James Woolfenden Date: Thu, 19 Jun 2025 19:38:13 +0100 Subject: [PATCH 2/3] Update src/data.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data.go b/src/data.go index 1d359b41..6013b013 100644 --- a/src/data.go +++ b/src/data.go @@ -167,7 +167,7 @@ func GetLocalModules(block *hclsyntax.Block, dirName string, listModulesJson Mod modulePath = ReturnLocalAddrFromSource(modulePath, listModulesJson) } // have the path to the module - modulePath = filepath.Join(dirName, "/", modulePath) + modulePath = filepath.Join(dirName, modulePath) // now process these extras ExtraFiles, err := GetTF(modulePath) if err != nil { From 8ef41e92f64201056b9a2bb57716e015f94030d5 Mon Sep 17 00:00:00 2001 From: James Woolfenden Date: Fri, 20 Jun 2025 18:31:03 +0100 Subject: [PATCH 3/3] added unit test file and fixed naming of new --- src/modules.go | 32 +++-- src/modules_test.go | 305 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 328 insertions(+), 9 deletions(-) create mode 100644 src/modules_test.go diff --git a/src/modules.go b/src/modules.go index 4103767a..a3f34b17 100644 --- a/src/modules.go +++ b/src/modules.go @@ -7,7 +7,7 @@ import ( "os" "path/filepath" - version "github.com/hashicorp/go-version" + "github.com/hashicorp/go-version" ) const ( @@ -15,7 +15,7 @@ const ( ) // Record represents some metadata about an installed module, as part -// of a modules json. +// of a module JSON. type Record struct { Key string `json:"Key"` SourceAddr string `json:"Source"` @@ -30,8 +30,19 @@ type modulesJson struct { Records []Record `json:"Modules"` } +type invalidVersionError struct { + err error + key string + version string +} + +func (m *invalidVersionError) Error() string { + return fmt.Sprintf("invalid version %q for %s: %s", m.version, m.key, m.err) +} + func ReadModuleJson(r io.Reader) (ModuleJson, error) { src, err := io.ReadAll(r) + if err != nil { return nil, err } @@ -42,27 +53,29 @@ func ReadModuleJson(r io.Reader) (ModuleJson, error) { var read modulesJson err = json.Unmarshal(src, &read) + if err != nil { - return nil, fmt.Errorf("error unmarshalling snapshot: %v", err) + return nil, &unmarshallJSONError{err, ""} } - new := make(ModuleJson) + + newModuleJson := make(ModuleJson) for _, record := range read.Records { if record.VersionStr != "" { record.Version, err = version.NewVersion(record.VersionStr) if err != nil { - return nil, fmt.Errorf("invalid version %q for %s: %s", record.VersionStr, record.Key, err) + return nil, &invalidVersionError{err, record.Key, record.VersionStr} } } // Ensure Windows is using the proper modules path format after - // reading the modules manifest Dir records + // reading the module's manifest Dir records record.Dir = filepath.FromSlash(record.Dir) - if _, exists := new[record.Key]; exists { + if _, exists := newModuleJson[record.Key]; exists { return nil, fmt.Errorf("snapshot file contains two records for path %s", record.Key) } - new[record.Key] = record + newModuleJson[record.Key] = record } - return new, nil + return newModuleJson, nil } func ReadModuleJsonForDir(dir string) (ModuleJson, error) { @@ -74,6 +87,7 @@ func ReadModuleJsonForDir(dir string) (ModuleJson, error) { } return nil, err } + defer r.Close() return ReadModuleJson(r) } diff --git a/src/modules_test.go b/src/modules_test.go new file mode 100644 index 00000000..da899d54 --- /dev/null +++ b/src/modules_test.go @@ -0,0 +1,305 @@ +package pike + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestReadModuleJson_ValidInput(t *testing.T) { + jsonInput := `{ + "Modules": [ + { + "Key": "module1", + "Source": "github.com/example/module1", + "Version": "1.0.0", + "Dir": "modules/module1" + }, + { + "Key": "module2", + "Source": "github.com/example/module2", + "Dir": "modules/module2" + } + ] + }` + + reader := strings.NewReader(jsonInput) + result, err := ReadModuleJson(reader) + + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + if len(result) != 2 { + t.Fatalf("Expected 2 modules, got: %d", len(result)) + } + + // Test module1 + module1, exists := result["module1"] + if !exists { + t.Fatal("Expected module1 to exist") + } + if module1.Key != "module1" { + t.Errorf("Expected Key 'module1', got: %s", module1.Key) + } + if module1.SourceAddr != "github.com/example/module1" { + t.Errorf("Expected SourceAddr 'github.com/example/module1', got: %s", module1.SourceAddr) + } + if module1.VersionStr != "1.0.0" { + t.Errorf("Expected VersionStr '1.0.0', got: %s", module1.VersionStr) + } + if module1.Version == nil { + t.Error("Expected Version to be parsed") + } else if module1.Version.String() != "1.0.0" { + t.Errorf("Expected Version '1.0.0', got: %s", module1.Version.String()) + } + + // Test module2 (no version) + module2, exists := result["module2"] + if !exists { + t.Fatal("Expected module2 to exist") + } + if module2.Version != nil { + t.Error("Expected Version to be nil for module without version") + } +} + +func TestReadModuleJson_EmptyInput(t *testing.T) { + reader := strings.NewReader("") + result, err := ReadModuleJson(reader) + + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + if len(result) != 0 { + t.Fatalf("Expected empty ModuleJson, got: %d modules", len(result)) + } +} + +func TestReadModuleJson_InvalidJSON(t *testing.T) { + reader := strings.NewReader(`{"invalid": json}`) + _, err := ReadModuleJson(reader) + + if err == nil { + t.Fatal("Expected error for invalid JSON") + } + + if !strings.Contains(err.Error(), "error unmarshalling snapshot") { + t.Errorf("Expected unmarshalling error, got: %v", err) + } +} + +func TestReadModuleJson_DuplicateKeys(t *testing.T) { + jsonInput := `{ + "Modules": [ + { + "Key": "duplicate", + "Source": "github.com/example/module1", + "Dir": "modules/module1" + }, + { + "Key": "duplicate", + "Source": "github.com/example/module2", + "Dir": "modules/module2" + } + ] + }` + + reader := strings.NewReader(jsonInput) + _, err := ReadModuleJson(reader) + + if err == nil { + t.Fatal("Expected error for duplicate keys") + } + + if !strings.Contains(err.Error(), "snapshot file contains two records for path duplicate") { + t.Errorf("Expected duplicate key error, got: %v", err) + } +} + +func TestReadModuleJson_InvalidVersion(t *testing.T) { + jsonInput := `{ + "Modules": [ + { + "Key": "module1", + "Source": "github.com/example/module1", + "Version": "invalid-version", + "Dir": "modules/module1" + } + ] + }` + + reader := strings.NewReader(jsonInput) + _, err := ReadModuleJson(reader) + + if err == nil { + t.Fatal("Expected error for invalid version") + } + + if !strings.Contains(err.Error(), "invalid version") { + t.Errorf("Expected invalid version error, got: %v", err) + } +} + +func TestReadModuleJson_WindowsPathHandling(t *testing.T) { + jsonInput := `{ + "Modules": [ + { + "Key": "module1", + "Source": "github.com/example/module1", + "Dir": "modules/subdir/module1" + } + ] + }` + + reader := strings.NewReader(jsonInput) + result, err := ReadModuleJson(reader) + + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + module1 := result["module1"] + expectedDir := filepath.FromSlash("modules/subdir/module1") + if module1.Dir != expectedDir { + t.Errorf("Expected Dir '%s', got: '%s'", expectedDir, module1.Dir) + } +} + +func TestReadModuleJsonForDir_ValidFile(t *testing.T) { + // Create a temporary directory + tempDir, err := os.MkdirTemp("", "pike-test") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tempDir) + + // Create a modules.json file + jsonContent := `{ + "Modules": [ + { + "Key": "test-module", + "Source": "github.com/example/test", + "Dir": "modules/test" + } + ] + }` + + modulesFile := filepath.Join(tempDir, ManifestSnapshotFilename) + err = os.WriteFile(modulesFile, []byte(jsonContent), 0644) + if err != nil { + t.Fatalf("Failed to write modules file: %v", err) + } + + result, err := ReadModuleJsonForDir(tempDir) + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + if len(result) != 1 { + t.Fatalf("Expected 1 module, got: %d", len(result)) + } + + module, exists := result["test-module"] + if !exists { + t.Fatal("Expected test-module to exist") + } + if module.SourceAddr != "github.com/example/test" { + t.Errorf("Expected SourceAddr 'github.com/example/test', got: %s", module.SourceAddr) + } +} + +func TestReadModuleJsonForDir_NonExistentFile(t *testing.T) { + // Create a temporary directory without modules.json + tempDir, err := os.MkdirTemp("", "pike-test") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tempDir) + + result, err := ReadModuleJsonForDir(tempDir) + if err != nil { + t.Fatalf("Expected no error for non-existent file, got: %v", err) + } + + if len(result) != 0 { + t.Fatalf("Expected empty ModuleJson, got: %d modules", len(result)) + } +} + +func TestReadModuleJsonForDir_NonExistentDirectory(t *testing.T) { + result, err := ReadModuleJsonForDir("/non/existent/directory") + if err != nil { + t.Fatalf("Expected no error for non-existent directory, got: %v", err) + } + + if len(result) != 0 { + t.Fatalf("Expected empty ModuleJson, got: %d modules", len(result)) + } +} + +func TestReturnLocalAddrFromSource_Found(t *testing.T) { + modules := ModuleJson{ + "module1": Record{ + Key: "module1", + SourceAddr: "github.com/example/module1", + Dir: "/path/to/module1", + }, + "module2": Record{ + Key: "module2", + SourceAddr: "github.com/example/module2", + Dir: "/path/to/module2", + }, + } + + result := ReturnLocalAddrFromSource("github.com/example/module2", modules) + expected := "/path/to/module2" + + if result != expected { + t.Errorf("Expected '%s', got: '%s'", expected, result) + } +} + +func TestReturnLocalAddrFromSource_NotFound(t *testing.T) { + modules := ModuleJson{ + "module1": Record{ + Key: "module1", + SourceAddr: "github.com/example/module1", + Dir: "/path/to/module1", + }, + } + + result := ReturnLocalAddrFromSource("github.com/example/nonexistent", modules) + + if result != "" { + t.Errorf("Expected empty string, got: '%s'", result) + } +} + +func TestReturnLocalAddrFromSource_EmptyModules(t *testing.T) { + modules := make(ModuleJson) + + result := ReturnLocalAddrFromSource("github.com/example/any", modules) + + if result != "" { + t.Errorf("Expected empty string, got: '%s'", result) + } +} + +func TestReturnLocalAddrFromSource_EmptySource(t *testing.T) { + modules := ModuleJson{ + "module1": Record{ + Key: "module1", + SourceAddr: "github.com/example/module1", + Dir: "/path/to/module1", + }, + } + + result := ReturnLocalAddrFromSource("", modules) + + if result != "" { + t.Errorf("Expected empty string, got: '%s'", result) + } +}