diff --git a/go.mod b/go.mod index 37311ca9..e4e7e273 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,6 @@ require ( golang.org/x/oauth2 v0.30.0 google.golang.org/api v0.240.0 gopkg.in/ini.v1 v1.67.0 - ) require ( diff --git a/src/coverage/aws.md b/src/coverage/aws.md index 608da6db..91a8c024 100644 --- a/src/coverage/aws.md +++ b/src/coverage/aws.md @@ -2,8 +2,8 @@ | Terraform | Coverage % | Resources | Total Resources | |------------|------------|-----------|-----------------| -| Resources | 96.33 | 1498 | 1555 | -| Datasource | 99.52 | 625 | 628 | +| Resources | 95.96 | 1498 | 1561 | +| Datasource | 100.00 | 628 | 628 | ```shell ./resource.ps1 aws_appsync_api @@ -25,12 +25,18 @@ ./resource.ps1 aws_cognito_managed_login_branding ./resource.ps1 aws_connect_phone_number_contact_flow_association ./resource.ps1 aws_controltower_baseline +./resource.ps1 aws_ec2_allowed_images_settings +./resource.ps1 aws_fis_target_account_configuration ./resource.ps1 aws_fsx_s3_access_point_attachment +./resource.ps1 aws_invoicing_invoice_unit ./resource.ps1 aws_lakeformation_identity_center_configuration ./resource.ps1 aws_lakeformation_lf_tag_expression ./resource.ps1 aws_nat_gateway_eip_association ./resource.ps1 aws_networkfirewall_firewall_transit_gateway_attachment_accepter ./resource.ps1 aws_networkfirewall_vpc_endpoint_association +./resource.ps1 aws_networkflowmonitor_monitor +./resource.ps1 aws_networkflowmonitor_scope +./resource.ps1 aws_observabilityadmin_centralization_rule_for_organization ./resource.ps1 aws_odb_cloud_autonomous_vm_cluster ./resource.ps1 aws_odb_cloud_exadata_infrastructure ./resource.ps1 aws_odb_cloud_vm_cluster @@ -63,7 +69,4 @@ ./resource.ps1 aws_workspacesweb_trust_store_association ./resource.ps1 aws_workspacesweb_user_access_logging_settings_association ./resource.ps1 aws_workspacesweb_user_settings_association -./resource.ps1 aws_ecrpublic_images -type data -./resource.ps1 aws_rds_global_cluster -type data -./resource.ps1 aws_vpn_connection -type data ``` diff --git a/src/coverage/azure.md b/src/coverage/azure.md index c1b4768c..c991e42e 100755 --- a/src/coverage/azure.md +++ b/src/coverage/azure.md @@ -2,8 +2,8 @@ | Terraform | Coverage % | Resources | Total Resources | |------------|------------|-----------|-----------------| -| Resources | 4.40 | 49 | 1113 | -| Datasource | 30.58 | 122 | 399 | +| Resources | 4.39 | 49 | 1115 | +| Datasource | 30.65 | 122 | 398 | ```shell ./resource.ps1 azurerm_aadb2c_directory @@ -65,6 +65,8 @@ ./resource.ps1 azurerm_api_management_tag ./resource.ps1 azurerm_api_management_user ./resource.ps1 azurerm_api_management_workspace +./resource.ps1 azurerm_api_management_workspace_api_version_set +./resource.ps1 azurerm_api_management_workspace_certificate ./resource.ps1 azurerm_api_management_workspace_policy ./resource.ps1 azurerm_api_management_workspace_policy_fragment ./resource.ps1 azurerm_app_configuration_feature @@ -578,6 +580,7 @@ ./resource.ps1 azurerm_mobile_network_site ./resource.ps1 azurerm_mobile_network_slice ./resource.ps1 azurerm_mongo_cluster +./resource.ps1 azurerm_mongo_cluster_firewall_rule ./resource.ps1 azurerm_monitor_aad_diagnostic_setting ./resource.ps1 azurerm_monitor_action_group ./resource.ps1 azurerm_monitor_activity_log_alert @@ -866,7 +869,6 @@ ./resource.ps1 azurerm_site_recovery_vmware_replication_policy_association ./resource.ps1 azurerm_snapshot ./resource.ps1 azurerm_source_control_token -./resource.ps1 azurerm_spatial_anchors_account ./resource.ps1 azurerm_spring_cloud_accelerator ./resource.ps1 azurerm_spring_cloud_active_deployment ./resource.ps1 azurerm_spring_cloud_api_portal @@ -1335,7 +1337,6 @@ ./resource.ps1 azurerm_servicebus_namespace_disaster_recovery_config -type data ./resource.ps1 azurerm_servicebus_topic_authorization_rule -type data ./resource.ps1 azurerm_site_recovery_replication_recovery_plan -type data -./resource.ps1 azurerm_spatial_anchors_account -type data ./resource.ps1 azurerm_stack_hci_storage_path -type data ./resource.ps1 azurerm_static_web_app -type data ./resource.ps1 azurerm_storage_queue -type data diff --git a/src/coverage/google.md b/src/coverage/google.md index 0ee35be9..2275cfd5 100755 --- a/src/coverage/google.md +++ b/src/coverage/google.md @@ -2,8 +2,8 @@ | Terraform | Coverage % | Resources | Total Resources | |------------|------------|-----------|-----------------| -| Resources | 62.94 | 783 | 1244 | -| Datasource | 99.75 | 398 | 399 | +| Resources | 62.39 | 783 | 1255 | +| Datasource | 99.01 | 399 | 403 | ```shell ./resource.ps1 google_access_context_manager_access_level_condition @@ -69,7 +69,12 @@ ./resource.ps1 google_certificate_manager_certificate_map ./resource.ps1 google_certificate_manager_certificate_map_entry ./resource.ps1 google_certificate_manager_trust_config +./resource.ps1 google_ces_agent ./resource.ps1 google_ces_app +./resource.ps1 google_ces_deployment +./resource.ps1 google_ces_example +./resource.ps1 google_ces_guardrail +./resource.ps1 google_ces_tool ./resource.ps1 google_ces_toolset ./resource.ps1 google_cloud_asset_folder_feed ./resource.ps1 google_cloud_asset_organization_feed @@ -79,6 +84,9 @@ ./resource.ps1 google_cloud_ids_endpoint ./resource.ps1 google_cloud_quotas_quota_adjuster_settings ./resource.ps1 google_cloud_quotas_quota_preference +./resource.ps1 google_cloud_security_compliance_cloud_control +./resource.ps1 google_cloud_security_compliance_framework +./resource.ps1 google_cloud_security_compliance_framework_deployment ./resource.ps1 google_cloud_tasks_queue ./resource.ps1 google_cloudbuild_bitbucket_server_config ./resource.ps1 google_cloudbuild_worker_pool @@ -151,6 +159,7 @@ ./resource.ps1 google_compute_region_commitment ./resource.ps1 google_compute_region_disk ./resource.ps1 google_compute_region_disk_resource_policy_attachment +./resource.ps1 google_compute_region_health_aggregation_policy ./resource.ps1 google_compute_region_instance_group_manager ./resource.ps1 google_compute_region_instance_template ./resource.ps1 google_compute_region_network_endpoint @@ -233,6 +242,7 @@ ./resource.ps1 google_discovery_engine_search_engine ./resource.ps1 google_discovery_engine_sitemap ./resource.ps1 google_discovery_engine_target_site +./resource.ps1 google_discovery_engine_user_store ./resource.ps1 google_dns_response_policy ./resource.ps1 google_dns_response_policy_rule ./resource.ps1 google_document_ai_processor @@ -401,6 +411,7 @@ ./resource.ps1 google_oracle_database_cloud_exadata_infrastructure ./resource.ps1 google_oracle_database_cloud_vm_cluster ./resource.ps1 google_oracle_database_db_system +./resource.ps1 google_oracle_database_exascale_db_storage_vault ./resource.ps1 google_oracle_database_odb_network ./resource.ps1 google_oracle_database_odb_subnet ./resource.ps1 google_org_policy_custom_constraint @@ -467,5 +478,8 @@ ./resource.ps1 google_storage_object_acl ./resource.ps1 google_storage_transfer_agent_pool ./resource.ps1 google_storage_transfer_job -./resource.ps1 google_artifact_registry_packages -type data +./resource.ps1 google_artifact_registry_python_packages -type data +./resource.ps1 google_cloud_identity_policy -type data +./resource.ps1 google_compute_reservation_block -type data +./resource.ps1 google_compute_reservation_sub_block -type data ``` diff --git a/src/files_gcp_datasource.go b/src/files_gcp_datasource.go index 516d0916..fdd48fc0 100644 --- a/src/files_gcp_datasource.go +++ b/src/files_gcp_datasource.go @@ -993,3 +993,6 @@ var dataGoogleComputeInterconnectLocation []byte //go:embed mapping/google/data/compute/google_compute_interconnect_locations.json var dataGoogleComputeInterconnectLocations []byte + +//go:embed mapping/google/data/artifactregistry/google_artifact_registry_packages.json +var dataGoogleArtifactRegistryPackages []byte diff --git a/src/gcp_datasource.go b/src/gcp_datasource.go index 3a5bdb64..56a9a275 100644 --- a/src/gcp_datasource.go +++ b/src/gcp_datasource.go @@ -422,6 +422,7 @@ func GCPDataLookup(result string) interface{} { "google_artifact_registry_npm_packages": dataGoogleArtifactRegistryNpmPackages, "google_compute_interconnect_location": dataGoogleComputeInterconnectLocation, "google_compute_interconnect_locations": dataGoogleComputeInterconnectLocations, + "google_artifact_registry_packages": dataGoogleArtifactRegistryPackages, } return TFLookup[result] diff --git a/src/gcp_policy_test.go b/src/gcp_policy_test.go index a2dabef4..16fdab3d 100644 --- a/src/gcp_policy_test.go +++ b/src/gcp_policy_test.go @@ -10,7 +10,7 @@ import ( func TestGCPPolicy(t *testing.T) { t.Parallel() - os.Setenv("GCP_PROJECT", "pike-412922") + _ = os.Setenv("GCP_PROJECT", "pike-412922") type args struct { permissions []string } @@ -66,9 +66,9 @@ func TestGetCurrentProject_EnvironmentVariables(t *testing.T) { // Clean up after test defer func() { - os.Setenv("GOOGLE_CLOUD_PROJECT", originalGoogleCloudProject) - os.Setenv("GOOGLE_PROJECT", originalGoogleProject) - os.Setenv("GCP_PROJECT", originalGcpProject) + _ = os.Setenv("GOOGLE_CLOUD_PROJECT", originalGoogleCloudProject) + _ = os.Setenv("GOOGLE_PROJECT", originalGoogleProject) + _ = os.Setenv("GCP_PROJECT", originalGcpProject) }() tests := []struct { @@ -101,19 +101,19 @@ func TestGetCurrentProject_EnvironmentVariables(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Clear all environment variables - os.Unsetenv("GOOGLE_CLOUD_PROJECT") - os.Unsetenv("GOOGLE_PROJECT") - os.Unsetenv("GCP_PROJECT") + _ = os.Unsetenv("GOOGLE_CLOUD_PROJECT") + _ = os.Unsetenv("GOOGLE_PROJECT") + _ = os.Unsetenv("GCP_PROJECT") // Set test values if tt.googleCloudProject != "" { - os.Setenv("GOOGLE_CLOUD_PROJECT", tt.googleCloudProject) + _ = os.Setenv("GOOGLE_CLOUD_PROJECT", tt.googleCloudProject) } if tt.googleProject != "" { - os.Setenv("GOOGLE_PROJECT", tt.googleProject) + _ = os.Setenv("GOOGLE_PROJECT", tt.googleProject) } if tt.gcpProject != "" { - os.Setenv("GCP_PROJECT", tt.gcpProject) + _ = os.Setenv("GCP_PROJECT", tt.gcpProject) } project, err := getCurrentProject() @@ -138,27 +138,27 @@ func TestGetCurrentProject_GcloudConfigFile(t *testing.T) { // Clean up after test defer func() { - os.Setenv("GOOGLE_CLOUD_PROJECT", originalGoogleCloudProject) - os.Setenv("GOOGLE_PROJECT", originalGoogleProject) - os.Setenv("GCP_PROJECT", originalGcpProject) - os.Setenv("HOME", originalHome) - os.Setenv("APPDATA", originalAppData) + _ = os.Setenv("GOOGLE_CLOUD_PROJECT", originalGoogleCloudProject) + _ = os.Setenv("GOOGLE_PROJECT", originalGoogleProject) + _ = os.Setenv("GCP_PROJECT", originalGcpProject) + _ = os.Setenv("HOME", originalHome) + _ = os.Setenv("APPDATA", originalAppData) }() // Clear environment variables to force config file reading - os.Unsetenv("GOOGLE_CLOUD_PROJECT") - os.Unsetenv("GOOGLE_PROJECT") - os.Unsetenv("GCP_PROJECT") + _ = os.Unsetenv("GOOGLE_CLOUD_PROJECT") + _ = os.Unsetenv("GOOGLE_PROJECT") + _ = os.Unsetenv("GCP_PROJECT") // Create temporary directory structure tempDir := t.TempDir() var configPath string if runtime.GOOS != "windows" { - os.Setenv("HOME", tempDir) + _ = os.Setenv("HOME", tempDir) configPath = filepath.Join(tempDir, ".config", "gcloud", "configurations", "config_default") } else { - os.Setenv("APPDATA", tempDir) + _ = os.Setenv("APPDATA", tempDir) configPath = filepath.Join(tempDir, "gcloud", "configurations", "config_default") } @@ -201,15 +201,15 @@ func TestGetCurrentProject_EmptyEnvironmentVariables(t *testing.T) { // Clean up after test defer func() { - os.Setenv("GOOGLE_CLOUD_PROJECT", originalGoogleCloudProject) - os.Setenv("GOOGLE_PROJECT", originalGoogleProject) - os.Setenv("GCP_PROJECT", originalGcpProject) + _ = os.Setenv("GOOGLE_CLOUD_PROJECT", originalGoogleCloudProject) + _ = os.Setenv("GOOGLE_PROJECT", originalGoogleProject) + _ = os.Setenv("GCP_PROJECT", originalGcpProject) }() // Test empty string environment variables (should be treated as not set) - os.Setenv("GOOGLE_CLOUD_PROJECT", "") - os.Setenv("GOOGLE_PROJECT", "") - os.Setenv("GCP_PROJECT", "") + _ = os.Setenv("GOOGLE_CLOUD_PROJECT", "") + _ = os.Setenv("GOOGLE_PROJECT", "") + _ = os.Setenv("GCP_PROJECT", "") // This will likely fail due to no credentials or config file, but we're testing the logic _, err := getCurrentProject() @@ -229,26 +229,26 @@ func TestGetCurrentProject_MissingConfigFile(t *testing.T) { // Clean up after test defer func() { - os.Setenv("GOOGLE_CLOUD_PROJECT", originalGoogleCloudProject) - os.Setenv("GOOGLE_PROJECT", originalGoogleProject) - os.Setenv("GCP_PROJECT", originalGcpProject) - os.Setenv("HOME", originalHome) - os.Setenv("APPDATA", originalAppData) + _ = os.Setenv("GOOGLE_CLOUD_PROJECT", originalGoogleCloudProject) + _ = os.Setenv("GOOGLE_PROJECT", originalGoogleProject) + _ = os.Setenv("GCP_PROJECT", originalGcpProject) + _ = os.Setenv("HOME", originalHome) + _ = os.Setenv("APPDATA", originalAppData) }() // Clear environment variables - os.Unsetenv("GOOGLE_CLOUD_PROJECT") - os.Unsetenv("GOOGLE_PROJECT") - os.Unsetenv("GCP_PROJECT") + _ = os.Unsetenv("GOOGLE_CLOUD_PROJECT") + _ = os.Unsetenv("GOOGLE_PROJECT") + _ = os.Unsetenv("GCP_PROJECT") // Set HOME/APPDATA to non-existent directory tempDir := t.TempDir() nonExistentDir := filepath.Join(tempDir, "nonexistent") if runtime.GOOS != "windows" { - os.Setenv("HOME", nonExistentDir) + _ = os.Setenv("HOME", nonExistentDir) } else { - os.Setenv("APPDATA", nonExistentDir) + _ = os.Setenv("APPDATA", nonExistentDir) } _, err := getCurrentProject() @@ -267,27 +267,27 @@ func TestGetCurrentProject_InvalidConfigFile(t *testing.T) { // Clean up after test defer func() { - os.Setenv("GOOGLE_CLOUD_PROJECT", originalGoogleCloudProject) - os.Setenv("GOOGLE_PROJECT", originalGoogleProject) - os.Setenv("GCP_PROJECT", originalGcpProject) - os.Setenv("HOME", originalHome) - os.Setenv("APPDATA", originalAppData) + _ = os.Setenv("GOOGLE_CLOUD_PROJECT", originalGoogleCloudProject) + _ = os.Setenv("GOOGLE_PROJECT", originalGoogleProject) + _ = os.Setenv("GCP_PROJECT", originalGcpProject) + _ = os.Setenv("HOME", originalHome) + _ = os.Setenv("APPDATA", originalAppData) }() // Clear environment variables - os.Unsetenv("GOOGLE_CLOUD_PROJECT") - os.Unsetenv("GOOGLE_PROJECT") - os.Unsetenv("GCP_PROJECT") + _ = os.Unsetenv("GOOGLE_CLOUD_PROJECT") + _ = os.Unsetenv("GOOGLE_PROJECT") + _ = os.Unsetenv("GCP_PROJECT") // Create temporary directory structure tempDir := t.TempDir() var configPath string if runtime.GOOS != "windows" { - os.Setenv("HOME", tempDir) + _ = os.Setenv("HOME", tempDir) configPath = filepath.Join(tempDir, ".config", "gcloud", "configurations", "config_default") } else { - os.Setenv("APPDATA", tempDir) + _ = os.Setenv("APPDATA", tempDir) configPath = filepath.Join(tempDir, "gcloud", "configurations", "config_default") } @@ -324,27 +324,27 @@ func TestGetCurrentProject_ConfigFileWithoutProject(t *testing.T) { // Clean up after test defer func() { - os.Setenv("GOOGLE_CLOUD_PROJECT", originalGoogleCloudProject) - os.Setenv("GOOGLE_PROJECT", originalGoogleProject) - os.Setenv("GCP_PROJECT", originalGcpProject) - os.Setenv("HOME", originalHome) - os.Setenv("APPDATA", originalAppData) + _ = os.Setenv("GOOGLE_CLOUD_PROJECT", originalGoogleCloudProject) + _ = os.Setenv("GOOGLE_PROJECT", originalGoogleProject) + _ = os.Setenv("GCP_PROJECT", originalGcpProject) + _ = os.Setenv("HOME", originalHome) + _ = os.Setenv("APPDATA", originalAppData) }() // Clear environment variables - os.Unsetenv("GOOGLE_CLOUD_PROJECT") - os.Unsetenv("GOOGLE_PROJECT") - os.Unsetenv("GCP_PROJECT") + _ = os.Unsetenv("GOOGLE_CLOUD_PROJECT") + _ = os.Unsetenv("GOOGLE_PROJECT") + _ = os.Unsetenv("GCP_PROJECT") // Create temporary directory structure tempDir := t.TempDir() var configPath string if runtime.GOOS != "windows" { - os.Setenv("HOME", tempDir) + _ = os.Setenv("HOME", tempDir) configPath = filepath.Join(tempDir, ".config", "gcloud", "configurations", "config_default") } else { - os.Setenv("APPDATA", tempDir) + _ = os.Setenv("APPDATA", tempDir) configPath = filepath.Join(tempDir, "gcloud", "configurations", "config_default") } diff --git a/src/mapping/google/resource/google_dataproc_gdc_spark_application.json b/src/mapping/google/data/artifactregistry/google_artifact_registry_packages.json similarity index 65% rename from src/mapping/google/resource/google_dataproc_gdc_spark_application.json rename to src/mapping/google/data/artifactregistry/google_artifact_registry_packages.json index 45e4fdaf..b84e9205 100644 --- a/src/mapping/google/resource/google_dataproc_gdc_spark_application.json +++ b/src/mapping/google/data/artifactregistry/google_artifact_registry_packages.json @@ -6,6 +6,8 @@ }, "destroy": [], "modify": [], - "plan": [] + "plan": [ + "artifactregistry.packages.list" + ] } ] diff --git a/src/mapping/google/resource/google_dataproc_gdc_application_environment.json b/src/mapping/google/data/google_cloud_identity_policy.json similarity index 100% rename from src/mapping/google/resource/google_dataproc_gdc_application_environment.json rename to src/mapping/google/data/google_cloud_identity_policy.json diff --git a/src/mapping/google/resource/google_dataproc_gdc_service_instance.json b/src/mapping/google/resource/google_apigee_environment.json similarity index 100% rename from src/mapping/google/resource/google_dataproc_gdc_service_instance.json rename to src/mapping/google/resource/google_apigee_environment.json diff --git a/src/mapping/google/resource/google_dataproc_metastore_federation.json b/src/mapping/google/resource/google_dataproc_metastore_federation.json deleted file mode 100644 index 45e4fdaf..00000000 --- a/src/mapping/google/resource/google_dataproc_metastore_federation.json +++ /dev/null @@ -1,11 +0,0 @@ -[ - { - "apply": [], - "attributes": { - "tags": [] - }, - "destroy": [], - "modify": [], - "plan": [] - } -] diff --git a/src/mapping/google/resource/google_dataproc_metastore_service.json b/src/mapping/google/resource/google_dataproc_metastore_service.json deleted file mode 100644 index 45e4fdaf..00000000 --- a/src/mapping/google/resource/google_dataproc_metastore_service.json +++ /dev/null @@ -1,11 +0,0 @@ -[ - { - "apply": [], - "attributes": { - "tags": [] - }, - "destroy": [], - "modify": [], - "plan": [] - } -] diff --git a/src/parse/aws-members.json b/src/parse/aws-members.json index 454c82b2..1c433ceb 100755 --- a/src/parse/aws-members.json +++ b/src/parse/aws-members.json @@ -489,6 +489,7 @@ "aws_ebs_snapshot_copy", "aws_ebs_snapshot_import", "aws_ebs_volume", + "aws_ec2_allowed_images_settings", "aws_ec2_availability_zone_group", "aws_ec2_capacity_block_reservation", "aws_ec2_capacity_reservation", @@ -620,6 +621,7 @@ "aws_finspace_kx_user", "aws_finspace_kx_volume", "aws_fis_experiment_template", + "aws_fis_target_account_configuration", "aws_flow_log", "aws_fms_admin_account", "aws_fms_policy", @@ -749,6 +751,7 @@ "aws_internet_gateway", "aws_internet_gateway_attachment", "aws_internetmonitor_monitor", + "aws_invoicing_invoice_unit", "aws_iot_authorizer", "aws_iot_billing_group", "aws_iot_ca_certificate", @@ -951,6 +954,8 @@ "aws_networkfirewall_rule_group", "aws_networkfirewall_tls_inspection_configuration", "aws_networkfirewall_vpc_endpoint_association", + "aws_networkflowmonitor_monitor", + "aws_networkflowmonitor_scope", "aws_networkmanager_attachment_accepter", "aws_networkmanager_connect_attachment", "aws_networkmanager_connect_peer", @@ -980,6 +985,7 @@ "aws_oam_link", "aws_oam_sink", "aws_oam_sink_policy", + "aws_observabilityadmin_centralization_rule_for_organization", "aws_odb_cloud_autonomous_vm_cluster", "aws_odb_cloud_exadata_infrastructure", "aws_odb_cloud_vm_cluster", diff --git a/src/parse/azurerm-members.json b/src/parse/azurerm-members.json index 4d530858..753b8536 100755 --- a/src/parse/azurerm-members.json +++ b/src/parse/azurerm-members.json @@ -60,6 +60,8 @@ "azurerm_api_management_tag", "azurerm_api_management_user", "azurerm_api_management_workspace", + "azurerm_api_management_workspace_api_version_set", + "azurerm_api_management_workspace_certificate", "azurerm_api_management_workspace_policy", "azurerm_api_management_workspace_policy_fragment", "azurerm_app_configuration", @@ -591,6 +593,7 @@ "azurerm_mobile_network_site", "azurerm_mobile_network_slice", "azurerm_mongo_cluster", + "azurerm_mongo_cluster_firewall_rule", "azurerm_monitor_aad_diagnostic_setting", "azurerm_monitor_action_group", "azurerm_monitor_activity_log_alert", @@ -894,7 +897,6 @@ "azurerm_site_recovery_vmware_replication_policy_association", "azurerm_snapshot", "azurerm_source_control_token", - "azurerm_spatial_anchors_account", "azurerm_spring_cloud_accelerator", "azurerm_spring_cloud_active_deployment", "azurerm_spring_cloud_api_portal", @@ -1458,7 +1460,6 @@ "azurerm_site_recovery_replication_recovery_plan", "azurerm_snapshot", "azurerm_source_control_token", - "azurerm_spatial_anchors_account", "azurerm_spring_cloud_app", "azurerm_spring_cloud_service", "azurerm_ssh_public_key", diff --git a/src/parse/google-members.json b/src/parse/google-members.json index 4bfbd906..ec006955 100755 --- a/src/parse/google-members.json +++ b/src/parse/google-members.json @@ -181,7 +181,12 @@ "google_certificate_manager_certificate_map_entry", "google_certificate_manager_dns_authorization", "google_certificate_manager_trust_config", + "google_ces_agent", "google_ces_app", + "google_ces_deployment", + "google_ces_example", + "google_ces_guardrail", + "google_ces_tool", "google_ces_toolset", "google_chronicle_data_access_label", "google_chronicle_data_access_scope", @@ -216,6 +221,9 @@ "google_cloud_run_v2_worker_pool_iam_member", "google_cloud_run_v2_worker_pool_iam_policy", "google_cloud_scheduler_job", + "google_cloud_security_compliance_cloud_control", + "google_cloud_security_compliance_framework", + "google_cloud_security_compliance_framework_deployment", "google_cloud_tasks_queue", "google_cloud_tasks_queue_iam_binding", "google_cloud_tasks_queue_iam_member", @@ -368,6 +376,7 @@ "google_compute_region_disk_iam_member", "google_compute_region_disk_iam_policy", "google_compute_region_disk_resource_policy_attachment", + "google_compute_region_health_aggregation_policy", "google_compute_region_health_check", "google_compute_region_instance_group_manager", "google_compute_region_instance_template", @@ -595,6 +604,7 @@ "google_discovery_engine_search_engine", "google_discovery_engine_sitemap", "google_discovery_engine_target_site", + "google_discovery_engine_user_store", "google_dns_managed_zone", "google_dns_managed_zone_iam_binding", "google_dns_managed_zone_iam_member", @@ -966,6 +976,7 @@ "google_oracle_database_cloud_exadata_infrastructure", "google_oracle_database_cloud_vm_cluster", "google_oracle_database_db_system", + "google_oracle_database_exascale_db_storage_vault", "google_oracle_database_odb_network", "google_oracle_database_odb_subnet", "google_org_policy_custom_constraint", @@ -1273,6 +1284,7 @@ "google_artifact_registry_npm_packages", "google_artifact_registry_packages", "google_artifact_registry_python_package", + "google_artifact_registry_python_packages", "google_artifact_registry_repositories", "google_artifact_registry_repository", "google_artifact_registry_repository_iam_policy", @@ -1317,6 +1329,7 @@ "google_cloud_identity_group_memberships", "google_cloud_identity_group_transitive_memberships", "google_cloud_identity_groups", + "google_cloud_identity_policy", "google_cloud_quotas_quota_info", "google_cloud_quotas_quota_infos", "google_cloud_run_locations", @@ -1392,6 +1405,8 @@ "google_compute_region_ssl_certificate", "google_compute_regions", "google_compute_reservation", + "google_compute_reservation_block", + "google_compute_reservation_sub_block", "google_compute_resource_policy", "google_compute_router", "google_compute_router_nat", diff --git a/src/scan.go b/src/scan.go index f7ca8117..6b6d9515 100644 --- a/src/scan.go +++ b/src/scan.go @@ -2,12 +2,15 @@ package pike import ( "context" + "errors" "fmt" "os" "os/exec" "path" "path/filepath" "strings" + "sync" + "time" "github.com/hashicorp/go-version" "github.com/hashicorp/hc-install/product" @@ -18,7 +21,16 @@ import ( const tfVersion = "1.5.4" -var dotTfModules = path.Join(".terraform", "modules") +const ( + modulesJSON = "modules.json" + dsStore = ".DS_Store" + dotTfModules = ".terraform/modules" +) + +var ( + terraformMutex sync.Mutex + initMutex sync.Map // per-directory mutex +) type emptyIACError struct{} @@ -184,23 +196,36 @@ func WriteOutput(outPolicy OutputPolicy, outputType string, scanPath string, out } // Init can download and install terraform if required and then terraform init your specified directory. + func Init(dirName string) (*string, []string, error) { + // Per-directory locking + dirMutex, _ := initMutex.LoadOrStore(dirName, &sync.Mutex{}) + mutex := dirMutex.(*sync.Mutex) + mutex.Lock() + defer mutex.Unlock() + tfPath, err := LocateTerraform() if err != nil { return nil, nil, &locateTerraformError{err} } tf, err := tfexec.NewTerraform(dirName, tfPath) + if err != nil { return nil, nil, &terraformExecError{err} } - err = tf.Init(context.Background(), tfexec.Upgrade(true)) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + err = tf.Init(ctx, tfexec.Upgrade(true)) if err != nil { + if errors.Is(context.DeadlineExceeded, ctx.Err()) { + return nil, nil, fmt.Errorf("terraform init timed out after 10 minutes: %w", err) + } return nil, nil, &terraformInitError{err} } - log.Printf("terraform init at %s", dirName) + log.Info().Msgf("terraform init at %s", dirName) modulesDir := path.Join(dirName, dotTfModules) modules, err := os.ReadDir(modulesDir) @@ -225,11 +250,14 @@ func Init(dirName string) (*string, []string, error) { // LocateTerraform finds the Terraform executable or installs it. func LocateTerraform() (string, error) { + terraformMutex.Lock() + defer terraformMutex.Unlock() + tfPath, err := exec.LookPath(terraform) // if you don't have tf installed, we have to install it if err != nil || tfPath == "" { - log.Printf("installing Terraform %s\n", tfVersion) + log.Info().Msgf("installing Terraform %s\n", tfVersion) installer := &releases.ExactVersion{ Product: product.Terraform, Version: version.Must(version.NewVersion(tfVersion)), @@ -247,18 +275,20 @@ func LocateTerraform() (string, error) { } // MakePolicy does the guts of determining a policy from code. -func MakePolicy(dirName string, file *string, init bool, EnableResources bool, provider string, policyName string) (OutputPolicy, error) { - var ( - output OutputPolicy - ) +func MakePolicy(dirName string, file *string, init bool, enableResources bool, provider string, policyName string) (OutputPolicy, error) { + // Validate inputs early + if dirName == "" && file == nil { + return OutputPolicy{}, errors.New("either directory or file should be be set") + } - permissionsBag, err := makePermissionBag(dirName, file, init, provider) + var output OutputPolicy + permissionsBag, err := makePermissionBag(dirName, file, init, provider) if err != nil { - return output, err + return output, fmt.Errorf("failed to create permission bag: %w", err) } - output, err = GetPolicy(permissionsBag, EnableResources, policyName) + output, err = GetPolicy(permissionsBag, enableResources, policyName) if err != nil { return output, &getPolicyError{err: err} } @@ -266,14 +296,22 @@ func MakePolicy(dirName string, file *string, init bool, EnableResources bool, p return output, nil } +// Extract common absolute path logic +func getAbsolutePath(path string) (string, error) { + absPath, err := filepath.Abs(path) + if err != nil { + return "", &absolutePathError{directory: path, err: err} + } + return absPath, nil +} func makePermissionBag(dirName string, file *string, init bool, provider string) (Sorted, error) { var files []string if file == nil { - fullPath, err := filepath.Abs(dirName) + fullPath, err := getAbsolutePath(dirName) if err != nil { - return Sorted{}, &absolutePathError{directory: dirName, err: err} + return Sorted{}, err } if init { @@ -292,9 +330,9 @@ func makePermissionBag(dirName string, file *string, init bool, provider string) return Sorted{}, &getTFError{directory: fullPath, err: err} } } else { - myFile, err := filepath.Abs(*file) + myFile, err := getAbsolutePath(*file) if err != nil { - return Sorted{}, &absolutePathError{directory: *file, err: err} + return Sorted{}, err } // is this a tfFile? @@ -310,22 +348,34 @@ func makePermissionBag(dirName string, file *string, init bool, provider string) } var resources []ResourceV2 + var failedFiles []string + var criticalErrors []error for _, tfFile := range files { resource, err := GetResources(tfFile, dirName) if err != nil { - // parse the other files - log.Print(err) + failedFiles = append(failedFiles, tfFile) + criticalErrors = append(criticalErrors, fmt.Errorf("failed to parse %s: %w", tfFile, err)) + continue } if resource != nil { resources = append(resources, resource...) } } + + // Fail fast if too many critical files failed + if len(criticalErrors) > 0 { + if len(failedFiles) > len(files)/2 { // More than 50% failed + return Sorted{}, fmt.Errorf("critical parsing failures in %d/%d files: %v", + len(failedFiles), len(files), criticalErrors) + } + log.Warn().Int("failed_files", len(failedFiles)).Msg("some terraform files failed to parse") + } + permissionsBag := GetPermissionBag(resources, provider) return permissionsBag, nil } - func GetPermissionBag(resources []ResourceV2, provider string) Sorted { var permissionBag Sorted var newPerms Sorted @@ -418,3 +468,16 @@ func StringInSlice(a string, list []string) bool { func GetHCLType(resourceName string) string { return strings.Split(resourceName, "_")[0] } + +const ( + maxFiles = 1000 + maxFileSize = 10 * 1024 * 1024 // 10MB + maxResources = 50000 +) + +func validateLimits(files []string) error { + if len(files) > maxFiles { + return fmt.Errorf("too many files: %d > %d", len(files), maxFiles) + } + return nil +} diff --git a/src/scan_test.go b/src/scan_test.go index 039b21e3..e575bda6 100644 --- a/src/scan_test.go +++ b/src/scan_test.go @@ -19,16 +19,16 @@ func TestScan(t *testing.T) { write bool } - testpath, _ := filepath.Abs("../terraform/aws/backup") + testPath, _ := filepath.Abs("../terraform/aws/backup") tests := []struct { name string args args wantErr bool }{ - {"aws", args{testpath, "json", false}, false}, - {"aws-out", args{testpath, "terraform", true}, false}, - {"google", args{testpath, "json", false}, false}, + {"aws", args{testPath, "json", false}, false}, + {"aws-out", args{testPath, "terraform", true}, false}, + {"google", args{testPath, "json", false}, false}, } for _, tt := range tests { diff --git a/terraform/google/backup/data.google_artifact_registry_packages.tf b/terraform/google/backup/data.google_artifact_registry_packages.tf new file mode 100644 index 00000000..c630d612 --- /dev/null +++ b/terraform/google/backup/data.google_artifact_registry_packages.tf @@ -0,0 +1,8 @@ +data "google_artifact_registry_packages" "pike" { + location = "us-central1" + repository_id = "pike" +} + +output "google_artifact_registry_packages" { + value = data.google_artifact_registry_packages.pike +} diff --git a/terraform/google/data.google_cloud_identity_policy.tf b/terraform/google/data.google_cloud_identity_policy.tf new file mode 100644 index 00000000..4c1f9ba1 --- /dev/null +++ b/terraform/google/data.google_cloud_identity_policy.tf @@ -0,0 +1,7 @@ +# data "google_cloud_identity_policy" "pike" { +# provider = google-beta +# } +# +# output "google_cloud_identity_policy" { +# value = data.google_cloud_identity_policy.pike +# } diff --git a/terraform/google/google_apigee_environment.tf b/terraform/google/google_apigee_environment.tf new file mode 100644 index 00000000..49c99f1b --- /dev/null +++ b/terraform/google/google_apigee_environment.tf @@ -0,0 +1,4 @@ +resource "google_apigee_environment" "pike" { + org_id = "pike" + name = "pike" +} diff --git a/terraform/google/provider.google.tf b/terraform/google/provider.google.tf index a89e890c..16af8596 100644 --- a/terraform/google/provider.google.tf +++ b/terraform/google/provider.google.tf @@ -1,23 +1,19 @@ provider "google" { - project = "pike-412922" - region = "europe-west2" - - # credentials = "C:\\Users\\jim_w\\pike-basic.json" - credentials = "/Users/jwoolfenden/pike-gcp-basic.json" + project = "pike-477416" + region = "europe-west2" + credentials = var.basic_gcp_credentials } provider "google-beta" { - project = "pike-412922" + project = "pike-477416" # region = "europe-west2" - region = "us-central1" - # credentials = "C:\\Users\\jim_w\\pike-basic.json" - credentials = "/Users/jwoolfenden/pike-gcp-basic.json" + region = "us-central1" + credentials = var.basic_gcp_credentials } provider "google" { - alias = "central" - project = "pike-412922" - region = "us-central1" - # credentials = "C:\\Users\\jim_w\\pike-basic.json" - credentials = "/Users/jwoolfenden/pike-gcp-basic.json" + alias = "central" + project = "pike-477416" + region = "us-central1" + credentials = var.basic_gcp_credentials } diff --git a/terraform/google/role/google_project_iam_binding.tf b/terraform/google/role/google_project_iam_binding.tf index 359fff60..63ccc883 100644 --- a/terraform/google/role/google_project_iam_binding.tf +++ b/terraform/google/role/google_project_iam_binding.tf @@ -1,5 +1,5 @@ resource "google_project_iam_binding" "pike" { - project = "pike-412922" + project = "pike-477416" role = google_project_iam_custom_role.terraform_pike.id members = [ diff --git a/terraform/google/role/google_project_iam_custom_role.tf b/terraform/google/role/google_project_iam_custom_role.tf index 9a45865a..f3e5d2c2 100644 --- a/terraform/google/role/google_project_iam_custom_role.tf +++ b/terraform/google/role/google_project_iam_custom_role.tf @@ -1,34 +1,10 @@ resource "google_project_iam_custom_role" "terraform_pike" { - project = "pike-412922" + project = "pike-477416" role_id = "terraform_pike" title = "terraform_pike" description = "A user with least privileges" permissions = [ - "aiplatform.indexEndpoints.create", - "aiplatform.indexes.create", - "compute.networks.get", - "iam.serviceAccounts.create", - "iam.serviceAccounts.delete", - "iam.serviceAccounts.get", - "iam.serviceAccounts.update", - "resourcemanager.projects.get", - "storage.buckets.create", - "storage.buckets.delete", - "storage.buckets.get", - "storage.buckets.update", - "storage.objects.create", - "storage.objects.delete", - "storage.objects.get", - "storage.objects.list", - - //new - //google_vertex_ai_endpoint_with_model_garden_deployment - "aiplatform.endpoints.create", - "aiplatform.endpoints.delete", - "aiplatform.endpoints.get", - "aiplatform.endpoints.update", - "aiplatform.endpoints.deploy", - "aiplatform.models.upload" + "artifactregistry.packages.list" ] } diff --git a/terraform/google/role/google_project_service.tf b/terraform/google/role/google_project_service.tf new file mode 100644 index 00000000..cd802888 --- /dev/null +++ b/terraform/google/role/google_project_service.tf @@ -0,0 +1,9 @@ +resource "google_project_service" "project" { + project = "pike-477416" + service = "iam.googleapis.com" +} + +resource "google_project_service" "cloudresources" { + project = "pike-477416" + service = "cloudresourcemanager.googleapis.com" +} diff --git a/terraform/google/role/google_service_account.tf b/terraform/google/role/google_service_account.tf index 44792caf..ee51e55a 100644 --- a/terraform/google/role/google_service_account.tf +++ b/terraform/google/role/google_service_account.tf @@ -1,5 +1,5 @@ resource "google_service_account" "pike" { account_id = "pike-service" display_name = "pike" - project = "pike-412922" + project = "pike-477416" } diff --git a/terraform/google/role/output.tf b/terraform/google/role/output.tf index b6efcf01..12aecc51 100644 --- a/terraform/google/role/output.tf +++ b/terraform/google/role/output.tf @@ -13,3 +13,7 @@ output "custom_role" { output "state_bucket" { value = google_storage_bucket.default } + +output "services" { + value = google_project_service.project +} diff --git a/terraform/google/role/provider.gcp.tf b/terraform/google/role/provider.gcp.tf index 9e7acaa5..de4978bc 100644 --- a/terraform/google/role/provider.gcp.tf +++ b/terraform/google/role/provider.gcp.tf @@ -1,5 +1,5 @@ provider "google" { - project = "pike-412922" + project = "pike-477416" # credentials = "c:/Users/jim_w/pike-412922-3277f720572e.json" credentials = "/Users/jwoolfenden/pike-gcp-super.json" } diff --git a/terraform/google/terraform.tf b/terraform/google/terraform.tf index b6e3eb00..e8aa3c32 100644 --- a/terraform/google/terraform.tf +++ b/terraform/google/terraform.tf @@ -1,8 +1,9 @@ terraform { backend "gcs" { - credentials = "/Users/jwoolfenden/pike-gcp-super.json" - # credentials = "c:/Users/jim_w/pike-412922-3277f720572e.json" - bucket = "terraform-pike-bucket-tfstate" - prefix = "trial/state" + + # credentials = "/Users/jwoolfenden/pike-gcp-super.json" + credentials = "c:/Users/jim_w/pike-412922-3277f720572e.json" + bucket = "terraform-pike-bucket-tfstate" + prefix = "trial/state" } } diff --git a/terraform/google/variables.tf b/terraform/google/variables.tf new file mode 100644 index 00000000..dd3d657a --- /dev/null +++ b/terraform/google/variables.tf @@ -0,0 +1,5 @@ +variable "basic_gcp_credentials" { + type = string + default = "C:\\Users\\jim_w\\pike-basic.json" + # default = "/Users/jwoolfenden/pike-gcp-basic.json" +}