From 6716847d03ed09aa4dfa7d80b0dd08c51e5e03f0 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 30 Aug 2024 07:52:33 +0100 Subject: [PATCH 01/81] chore: update the golden tests [IC-1572] (#3185) * chore: update the golden tests * chore: update golden tests The latest AzureRM provider (4.0.1) removes a number of resources and changes a number. - update tests terraform where appropriate/applicable - ignore CLI tests where the provider change doesn't make sense to rework the test * fix: incorrect default sku for pubic ip --- .../eks_cluster_test/eks_cluster_test.golden | 6 +- .../azure/app_service_environment_test.go | 6 +- ...actory_integration_runtime_managed_test.go | 5 +- .../integration_service_environment_test.go | 6 +- .../terraform/azure/lb_outbound_rule_test.go | 13 ++++- .../providers/terraform/azure/lb_rule_test.go | 12 +++- internal/providers/terraform/azure/lb_test.go | 7 ++- .../terraform/azure/mariadb_server_test.go | 6 +- .../terraform/azure/mysql_server_test.go | 2 + .../providers/terraform/azure/public_ip.go | 9 ++- .../terraform/azure/sql_database_test.go | 2 + .../terraform/azure/sql_elasticpool_test.go | 2 + .../azure/sql_managed_instance_test.go | 6 +- .../application_gateway_test.golden | 6 +- .../cognitive_account_test.golden | 14 ++--- .../cognitive_deployment_test.golden | 9 ++- .../cognitive_deployment_test.tf | 33 +++++------ .../cosmosdb_cassandra_keyspace_test.golden | 8 +-- .../cosmosdb_cassandra_keyspace_test.tf | 10 ++-- ...yspace_test_with_blank_geo_location.golden | 2 +- ...a_keyspace_test_with_blank_geo_location.tf | 10 ++-- .../cosmosdb_cassandra_table_test.golden | 12 ++-- .../cosmosdb_cassandra_table_test.tf | 10 ++-- .../cosmosdb_gremlin_database_test.golden | 8 +-- .../cosmosdb_gremlin_database_test.tf | 10 ++-- .../cosmosdb_gremlin_graph_test.golden | 8 +-- .../cosmosdb_gremlin_graph_test.tf | 10 ++-- .../cosmosdb_mongo_collection_test.golden | 12 ++-- .../cosmosdb_mongo_collection_test.tf | 10 ++-- .../cosmosdb_mongo_database_test.golden | 8 +-- .../cosmosdb_mongo_database_test.tf | 10 ++-- .../cosmosdb_sql_container_test.tf | 20 +++---- .../cosmosdb_sql_database_test.golden | 8 +-- .../cosmosdb_sql_database_test.tf | 10 ++-- .../cosmosdb_table_test.golden | 8 +-- .../cosmosdb_table_test.tf | 10 ++-- .../event_hubs_namespace_test.tf | 6 +- .../kubernetes_cluster_test.golden | 6 +- .../lb_outbound_rule_test.golden | 6 +- .../lb_outbound_rule_v2_test.golden | 6 +- .../testdata/lb_rule_test/lb_rule_test.golden | 10 ++-- .../lb_rule_v2_test/lb_rule_v2_test.golden | 12 ++-- .../lb_rule_v2_test/lb_rule_v2_test.tf | 3 + .../log_analytics_workspace_test.golden | 4 +- .../log_analytics_workspace_test.tf | 7 --- ...hine_learning_compute_instance_test.golden | 44 +++++++------- .../machine_learning_compute_instance_test.tf | 2 - .../monitor_data_collection_rule_test.tf | 1 + .../mssql_database_test.tf | 38 ++++++------ .../mssql_elasticpool_test.tf | 18 +++--- .../public_ip_test/public_ip_test.golden | 10 ++-- .../service_plan_test.golden | 58 +++++++++---------- .../synapse_spark_pool_test.tf | 8 +-- .../synapse_sql_pool_test.tf | 9 +-- .../synapse_workspace_test.tf | 26 --------- .../virtual_machine_test.golden | 16 ++--- ...ual_network_gateway_connection_test.golden | 6 +- .../virtual_network_gateway_test.golden | 6 +- .../cloudfunctions_function_test.golden | 32 +++++----- 59 files changed, 337 insertions(+), 325 deletions(-) diff --git a/internal/providers/terraform/aws/testdata/eks_cluster_test/eks_cluster_test.golden b/internal/providers/terraform/aws/testdata/eks_cluster_test/eks_cluster_test.golden index 329a6796d24..c5eebf46947 100644 --- a/internal/providers/terraform/aws/testdata/eks_cluster_test/eks_cluster_test.golden +++ b/internal/providers/terraform/aws/testdata/eks_cluster_test/eks_cluster_test.golden @@ -14,7 +14,7 @@ └─ EKS cluster (extended support) 730 hours $438.00 aws_eks_cluster.extended_support["1.27"] - └─ EKS cluster 730 hours $73.00 + └─ EKS cluster (extended support) 730 hours $438.00 aws_eks_cluster.extended_support["1.28"] └─ EKS cluster 730 hours $73.00 @@ -25,7 +25,7 @@ aws_eks_cluster.my_aws_eks_cluster └─ EKS cluster 730 hours $73.00 - OVERALL TOTAL $2,044.00 + OVERALL TOTAL $2,409.00 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -36,5 +36,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $2,044 ┃ - ┃ $2,044 ┃ +┃ main ┃ $2,409 ┃ - ┃ $2,409 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/app_service_environment_test.go b/internal/providers/terraform/azure/app_service_environment_test.go index be288ccb838..d41d6d7e06b 100644 --- a/internal/providers/terraform/azure/app_service_environment_test.go +++ b/internal/providers/terraform/azure/app_service_environment_test.go @@ -12,5 +12,9 @@ func TestAzureRMAppIsolatedServicePlan(t *testing.T) { t.Skip("skipping test in short mode") } - tftest.GoldenFileResourceTests(t, "app_service_environment_test") + opts := tftest.DefaultGoldenFileOptions() + // Ignore the CLI because the resource has been removed from the provider + opts.IgnoreCLI = true + + tftest.GoldenFileResourceTestsWithOpts(t, "app_service_environment_test", opts) } diff --git a/internal/providers/terraform/azure/data_factory_integration_runtime_managed_test.go b/internal/providers/terraform/azure/data_factory_integration_runtime_managed_test.go index 392d4d7d6ee..26dbb8e19f8 100644 --- a/internal/providers/terraform/azure/data_factory_integration_runtime_managed_test.go +++ b/internal/providers/terraform/azure/data_factory_integration_runtime_managed_test.go @@ -11,6 +11,9 @@ func TestDataFactoryIntegrationRuntimeManaged(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode") } + opts := tftest.DefaultGoldenFileOptions() + // Ignore the CLI because the resource has been removed from the provider + opts.IgnoreCLI = true - tftest.GoldenFileResourceTests(t, "data_factory_integration_runtime_managed_test") + tftest.GoldenFileResourceTestsWithOpts(t, "data_factory_integration_runtime_managed_test", opts) } diff --git a/internal/providers/terraform/azure/integration_service_environment_test.go b/internal/providers/terraform/azure/integration_service_environment_test.go index 4e9fed26169..57c318e1d28 100644 --- a/internal/providers/terraform/azure/integration_service_environment_test.go +++ b/internal/providers/terraform/azure/integration_service_environment_test.go @@ -12,5 +12,9 @@ func TestAzureRMAIntegrationServiceEnvironment(t *testing.T) { t.Skip("skipping test in short mode") } - tftest.GoldenFileResourceTests(t, "integration_service_environment_test") + opts := tftest.DefaultGoldenFileOptions() + // Ignore the CLI because the resource has been removed from the provider + opts.IgnoreCLI = true + + tftest.GoldenFileResourceTestsWithOpts(t, "integration_service_environment_test", opts) } diff --git a/internal/providers/terraform/azure/lb_outbound_rule_test.go b/internal/providers/terraform/azure/lb_outbound_rule_test.go index 77bf7b8315c..e10308fd104 100644 --- a/internal/providers/terraform/azure/lb_outbound_rule_test.go +++ b/internal/providers/terraform/azure/lb_outbound_rule_test.go @@ -11,8 +11,12 @@ func TestAzureRMLoadBalancerOutboundRuleGoldenFile(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode") } + opts := tftest.DefaultGoldenFileOptions() - tftest.GoldenFileResourceTests(t, "lb_outbound_rule_test") + // Skip CLI diff as it yields different results + opts.IgnoreCLI = true + + tftest.GoldenFileResourceTestsWithOpts(t, "lb_outbound_rule_test", opts) } func TestAzureRMLoadBalancerOutboundRuleV2GoldenFile(t *testing.T) { @@ -21,5 +25,10 @@ func TestAzureRMLoadBalancerOutboundRuleV2GoldenFile(t *testing.T) { t.Skip("skipping test in short mode") } - tftest.GoldenFileResourceTests(t, "lb_outbound_rule_v2_test") + opts := tftest.DefaultGoldenFileOptions() + + // Skip CLI diff as it yields different results + opts.IgnoreCLI = true + + tftest.GoldenFileResourceTestsWithOpts(t, "lb_outbound_rule_v2_test", opts) } diff --git a/internal/providers/terraform/azure/lb_rule_test.go b/internal/providers/terraform/azure/lb_rule_test.go index aa7849ef9b1..d509beefe1b 100644 --- a/internal/providers/terraform/azure/lb_rule_test.go +++ b/internal/providers/terraform/azure/lb_rule_test.go @@ -12,7 +12,11 @@ func TestAzureRMLoadBalancerRuleGoldenFile(t *testing.T) { t.Skip("skipping test in short mode") } - tftest.GoldenFileResourceTests(t, "lb_rule_test") + opts := tftest.DefaultGoldenFileOptions() + // Skip CLI diff as it yields different results + opts.IgnoreCLI = true + + tftest.GoldenFileResourceTestsWithOpts(t, "lb_rule_test", opts) } func TestAzureRMLoadBalancerRuleV2GoldenFile(t *testing.T) { @@ -21,5 +25,9 @@ func TestAzureRMLoadBalancerRuleV2GoldenFile(t *testing.T) { t.Skip("skipping test in short mode") } - tftest.GoldenFileResourceTests(t, "lb_rule_v2_test") + opts := tftest.DefaultGoldenFileOptions() + // Skip CLI diff as it yields different results + opts.IgnoreCLI = true + + tftest.GoldenFileResourceTestsWithOpts(t, "lb_rule_v2_test", opts) } diff --git a/internal/providers/terraform/azure/lb_test.go b/internal/providers/terraform/azure/lb_test.go index 3238ec9b9d2..95876203ffe 100644 --- a/internal/providers/terraform/azure/lb_test.go +++ b/internal/providers/terraform/azure/lb_test.go @@ -12,5 +12,10 @@ func TestAzureRMLoadBalancerGoldenFile(t *testing.T) { t.Skip("skipping test in short mode") } - tftest.GoldenFileResourceTests(t, "lb_test") + opts := tftest.DefaultGoldenFileOptions() + + // Skip CLI diff as it yields different results + opts.IgnoreCLI = true + + tftest.GoldenFileResourceTestsWithOpts(t, "lb_test", opts) } diff --git a/internal/providers/terraform/azure/mariadb_server_test.go b/internal/providers/terraform/azure/mariadb_server_test.go index 549c1aec12f..b81b25e5216 100644 --- a/internal/providers/terraform/azure/mariadb_server_test.go +++ b/internal/providers/terraform/azure/mariadb_server_test.go @@ -12,5 +12,9 @@ func TestMariaDBServer(t *testing.T) { t.Skip("skipping test in short mode") } - tftest.GoldenFileResourceTests(t, "mariadb_server_test") + opts := tftest.DefaultGoldenFileOptions() + // Ignore the CLI because the resource has been removed from the provider in favour of azurerm_mysql_flexible_server + opts.IgnoreCLI = true + + tftest.GoldenFileResourceTestsWithOpts(t, "mariadb_server_test", opts) } diff --git a/internal/providers/terraform/azure/mysql_server_test.go b/internal/providers/terraform/azure/mysql_server_test.go index 6b22fb69f63..53e4d1ba300 100644 --- a/internal/providers/terraform/azure/mysql_server_test.go +++ b/internal/providers/terraform/azure/mysql_server_test.go @@ -13,5 +13,7 @@ func TestMySQLServer_usage(t *testing.T) { opts := tftest.DefaultGoldenFileOptions() opts.CaptureLogs = true + // ignore CLI - this has been removed from the latest provider + opts.IgnoreCLI = true tftest.GoldenFileResourceTestsWithOpts(t, "mysql_server_test", opts) } diff --git a/internal/providers/terraform/azure/public_ip.go b/internal/providers/terraform/azure/public_ip.go index 570cfbe5419..e606da05306 100644 --- a/internal/providers/terraform/azure/public_ip.go +++ b/internal/providers/terraform/azure/public_ip.go @@ -21,15 +21,14 @@ func NewAzureRMPublicIP(d *schema.ResourceData, u *schema.UsageData) *schema.Res region := d.Region var meterName string - sku := "Basic" + sku := "Standard" // default sku is Standard + skuTier := "Regional" allocationMethod := d.Get("allocation_method").String() if d.Get("sku").Type != gjson.Null { sku = d.Get("sku").String() } - skuTier := "regional" - switch sku { case "Basic": meterName = "Basic IPv4 " + allocationMethod + " Public IP" @@ -38,8 +37,8 @@ func NewAzureRMPublicIP(d *schema.ResourceData, u *schema.UsageData) *schema.Res if skuTierVal != "" { skuTier = skuTierVal } - if skuTier == "global" { - sku = "Global" + if skuTier == "Global" { + sku = "Standard" // When sku_tier is Global, sku is Standard meterName = "Global IPv4 " + allocationMethod + " Public IP" } else { meterName = "Standard IPv4 " + allocationMethod + " Public IP" diff --git a/internal/providers/terraform/azure/sql_database_test.go b/internal/providers/terraform/azure/sql_database_test.go index 6b7e7c067b4..e5fa4c179f8 100644 --- a/internal/providers/terraform/azure/sql_database_test.go +++ b/internal/providers/terraform/azure/sql_database_test.go @@ -13,5 +13,7 @@ func TestSQLDatabase(t *testing.T) { opts := tftest.DefaultGoldenFileOptions() opts.CaptureLogs = true + // ignore CLI - this has been removed from the latest provider + opts.IgnoreCLI = true tftest.GoldenFileResourceTestsWithOpts(t, "sql_database_test", opts) } diff --git a/internal/providers/terraform/azure/sql_elasticpool_test.go b/internal/providers/terraform/azure/sql_elasticpool_test.go index 8f6e640db61..646cda85b1e 100644 --- a/internal/providers/terraform/azure/sql_elasticpool_test.go +++ b/internal/providers/terraform/azure/sql_elasticpool_test.go @@ -13,5 +13,7 @@ func TestSQLElasticPool(t *testing.T) { opts := tftest.DefaultGoldenFileOptions() opts.CaptureLogs = true + // ignore CLI - this has been removed from the latest provider + opts.IgnoreCLI = true tftest.GoldenFileResourceTestsWithOpts(t, "sql_elasticpool_test", opts) } diff --git a/internal/providers/terraform/azure/sql_managed_instance_test.go b/internal/providers/terraform/azure/sql_managed_instance_test.go index 30fcda62266..4ef4cabc462 100644 --- a/internal/providers/terraform/azure/sql_managed_instance_test.go +++ b/internal/providers/terraform/azure/sql_managed_instance_test.go @@ -12,5 +12,9 @@ func TestSQLManagedInstance(t *testing.T) { t.Skip("skipping test in short mode") } - tftest.GoldenFileResourceTests(t, "sql_managed_instance_test") + opts := tftest.DefaultGoldenFileOptions() + // This resource is removed from the latest provider + opts.IgnoreCLI = true + + tftest.GoldenFileResourceTestsWithOpts(t, "sql_managed_instance_test", opts) } diff --git a/internal/providers/terraform/azure/testdata/application_gateway_test/application_gateway_test.golden b/internal/providers/terraform/azure/testdata/application_gateway_test/application_gateway_test.golden index c048384c135..da18e697d91 100644 --- a/internal/providers/terraform/azure/testdata/application_gateway_test/application_gateway_test.golden +++ b/internal/providers/terraform/azure/testdata/application_gateway_test/application_gateway_test.golden @@ -42,9 +42,9 @@ └─ Data processing (0-10TB) Monthly cost depends on usage: $0.00 per GB azurerm_public_ip.example - └─ IP address (dynamic, regional) 730 hours $2.92 + └─ IP address (dynamic, regional) 730 hours not found - OVERALL TOTAL $5,914.54 + OVERALL TOTAL $5,911.62 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -56,5 +56,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $3,574 ┃ $2,340 ┃ $5,915 ┃ +┃ main ┃ $3,571 ┃ $2,340 ┃ $5,912 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/cognitive_account_test/cognitive_account_test.golden b/internal/providers/terraform/azure/testdata/cognitive_account_test/cognitive_account_test.golden index d8cc110f351..11e45b1303f 100644 --- a/internal/providers/terraform/azure/testdata/cognitive_account_test/cognitive_account_test.golden +++ b/internal/providers/terraform/azure/testdata/cognitive_account_test/cognitive_account_test.golden @@ -178,9 +178,9 @@ └─ Text analytics for health (5K-500K) Monthly cost depends on usage: $25.00 per 1K records azurerm_cognitive_account.with_usage["SpeechServices-S0"] - ├─ Speech to text 20 hours $20.00 + ├─ Speech to text 20 hours not found ├─ Speech to text batch 56 hours $10.08 - ├─ Speech to text custom model 17 hours $20.40 + ├─ Speech to text custom model 17 hours not found ├─ Speech to text custom model batch 44 hours $9.90 ├─ Speech to text custom endpoint hosting 372 hours $20.00 ├─ Speech to text custom training 2 hours $20.00 @@ -218,7 +218,7 @@ azurerm_cognitive_account.speech_with_commitment["invalid"] ├─ Speech to text batch Monthly cost depends on usage: $0.18 per hours - ├─ Speech to text custom model Monthly cost depends on usage: $1.20 per hours + ├─ Speech to text custom model not found ├─ Speech to text custom model batch Monthly cost depends on usage: $0.23 per hours ├─ Speech to text custom endpoint hosting Monthly cost depends on usage: $0.05375 per hours ├─ Speech to text custom training Monthly cost depends on usage: $10.00 per hours @@ -250,9 +250,9 @@ └─ Speech requests Monthly cost depends on usage: $5.50 per 1K transactions azurerm_cognitive_account.without_usage["SpeechServices-S0"] - ├─ Speech to text Monthly cost depends on usage: $1.00 per hours + ├─ Speech to text not found ├─ Speech to text batch Monthly cost depends on usage: $0.18 per hours - ├─ Speech to text custom model Monthly cost depends on usage: $1.20 per hours + ├─ Speech to text custom model not found ├─ Speech to text custom model batch Monthly cost depends on usage: $0.23 per hours ├─ Speech to text custom endpoint hosting Monthly cost depends on usage: $0.05375 per hours ├─ Speech to text custom training Monthly cost depends on usage: $10.00 per hours @@ -281,7 +281,7 @@ ├─ Customized training Monthly cost depends on usage: $3.00 per hour └─ Text analytics for health (5K-500K) Monthly cost depends on usage: $25.00 per 1K records - OVERALL TOTAL $314,876.75 + OVERALL TOTAL $314,836.35 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -295,7 +295,7 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $314,877 ┃ - ┃ $314,877 ┃ +┃ main ┃ $314,836 ┃ - ┃ $314,836 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ Logs: diff --git a/internal/providers/terraform/azure/testdata/cognitive_deployment_test/cognitive_deployment_test.golden b/internal/providers/terraform/azure/testdata/cognitive_deployment_test/cognitive_deployment_test.golden index be7e7007b6c..a698b1fc55a 100644 --- a/internal/providers/terraform/azure/testdata/cognitive_deployment_test/cognitive_deployment_test.golden +++ b/internal/providers/terraform/azure/testdata/cognitive_deployment_test/cognitive_deployment_test.golden @@ -315,6 +315,11 @@ azurerm_cognitive_deployment.eastus_without_usage["whisper"] └─ Text to speech (whisper) Monthly cost depends on usage: $0.36 per hours + azurerm_cognitive_deployment.free_tier + ├─ Language input (gpt-4) Monthly cost depends on usage: $0.03 per 1K tokens + ├─ Language output (gpt-4) Monthly cost depends on usage: $0.06 per 1K tokens + └─ Code interpreter sessions (gpt-4) Monthly cost depends on usage: $0.03 per sessions + azurerm_cognitive_deployment.swedencentral_without_usage["dall-e-3"] ├─ Standard 1024x1024 images (dall-e-3) Monthly cost depends on usage: $4.00 per 100 images ├─ Standard 1024x1792 images (dall-e-3) Monthly cost depends on usage: $8.00 per 100 images @@ -377,8 +382,8 @@ ────────────────────────────────── 116 cloud resources were detected: -∙ 108 were estimated -∙ 7 were free +∙ 109 were estimated +∙ 6 were free ∙ 1 is not supported yet, see https://infracost.io/requested-resources: ∙ 1 x azurerm_cognitive_deployment diff --git a/internal/providers/terraform/azure/testdata/cognitive_deployment_test/cognitive_deployment_test.tf b/internal/providers/terraform/azure/testdata/cognitive_deployment_test/cognitive_deployment_test.tf index 31b43587d02..f156ede2c30 100644 --- a/internal/providers/terraform/azure/testdata/cognitive_deployment_test/cognitive_deployment_test.tf +++ b/internal/providers/terraform/azure/testdata/cognitive_deployment_test/cognitive_deployment_test.tf @@ -96,8 +96,8 @@ resource "azurerm_cognitive_deployment" "eastus_without_usage" { version = each.value.version } - scale { - type = "Standard" + sku { + name = "Standard" } } @@ -113,8 +113,8 @@ resource "azurerm_cognitive_deployment" "eastus2_without_usage" { version = each.value.version } - scale { - type = "Standard" + sku { + name = "Standard" } } @@ -130,8 +130,8 @@ resource "azurerm_cognitive_deployment" "swedencentral_without_usage" { version = each.value.version } - scale { - type = "Standard" + sku { + name = "Standard" } } @@ -147,8 +147,8 @@ resource "azurerm_cognitive_deployment" "eastus_with_usage" { version = each.value.version } - scale { - type = "Standard" + sku { + name = "Standard" } } @@ -164,8 +164,8 @@ resource "azurerm_cognitive_deployment" "eastus2_with_usage" { version = each.value.version } - scale { - type = "Standard" + sku { + name = "Standard" } } @@ -181,8 +181,8 @@ resource "azurerm_cognitive_deployment" "swedencentral_with_usage" { version = each.value.version } - scale { - type = "Standard" + sku { + name = "Standard" } } @@ -195,9 +195,8 @@ resource "azurerm_cognitive_deployment" "free_tier" { name = "gpt-4" } - scale { - type = "Standard" - tier = "Free" + sku { + name = "Standard" } } @@ -210,7 +209,7 @@ resource "azurerm_cognitive_deployment" "unsupported" { name = "ada" } - scale { - type = "Standard" + sku { + name = "Standard" } } diff --git a/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_keyspace_test/cosmosdb_cassandra_keyspace_test.golden b/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_keyspace_test/cosmosdb_cassandra_keyspace_test.golden index 61211604922..043b8e21a0e 100644 --- a/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_keyspace_test/cosmosdb_cassandra_keyspace_test.golden +++ b/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_keyspace_test/cosmosdb_cassandra_keyspace_test.golden @@ -29,8 +29,8 @@ └─ Restored data 3,000 GB $450.00 azurerm_cosmosdb_cassandra_keyspace.mutli-master_backup2copies - ├─ Provisioned throughput (provisioned, West US) 10 RU/s x 100 $116.80 - ├─ Provisioned throughput (provisioned, Central US) 10 RU/s x 100 $116.80 + ├─ Provisioned throughput (provisioned, West US) 10 RU/s x 100 $58.40 + ├─ Provisioned throughput (provisioned, Central US) 10 RU/s x 100 $58.40 ├─ Transactional storage (West US) 1,000 GB $250.00 ├─ Transactional storage (Central US) 1,000 GB $250.00 └─ Restored data Monthly cost depends on usage: $0.15 per GB @@ -44,7 +44,7 @@ ├─ Periodic backup (West US) Monthly cost depends on usage: $0.15 per GB └─ Restored data Monthly cost depends on usage: $0.15 per GB - OVERALL TOTAL $5,155.73 + OVERALL TOTAL $5,038.93 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -56,5 +56,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $5,156 ┃ - ┃ $5,156 ┃ +┃ main ┃ $5,039 ┃ - ┃ $5,039 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_keyspace_test/cosmosdb_cassandra_keyspace_test.tf b/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_keyspace_test/cosmosdb_cassandra_keyspace_test.tf index aae98bfe771..c9db8d624a0 100644 --- a/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_keyspace_test/cosmosdb_cassandra_keyspace_test.tf +++ b/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_keyspace_test/cosmosdb_cassandra_keyspace_test.tf @@ -57,11 +57,11 @@ resource "azurerm_cosmosdb_account" "continuous_backup" { } resource "azurerm_cosmosdb_account" "multi-master_backup2copies" { - name = "tfex-cosmosdb-account" - resource_group_name = azurerm_resource_group.example.name - location = azurerm_resource_group.example.location - offer_type = "Standard" - enable_multiple_write_locations = true + name = "tfex-cosmosdb-account" + resource_group_name = azurerm_resource_group.example.name + location = azurerm_resource_group.example.location + offer_type = "Standard" + consistency_policy { consistency_level = "Strong" diff --git a/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_keyspace_test_with_blank_geo_location/cosmosdb_cassandra_keyspace_test_with_blank_geo_location.golden b/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_keyspace_test_with_blank_geo_location/cosmosdb_cassandra_keyspace_test_with_blank_geo_location.golden index 4db502078ac..0f794b4f5c9 100644 --- a/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_keyspace_test_with_blank_geo_location/cosmosdb_cassandra_keyspace_test_with_blank_geo_location.golden +++ b/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_keyspace_test_with_blank_geo_location/cosmosdb_cassandra_keyspace_test_with_blank_geo_location.golden @@ -2,7 +2,7 @@ Name Monthly Qty Unit Monthly Cost azurerm_cosmosdb_cassandra_keyspace.with_blank_geo_location - ├─ Provisioned throughput (autoscale, East US) Monthly cost depends on usage: $11.68 per RU/s x 100 + ├─ Provisioned throughput (autoscale, East US) Monthly cost depends on usage: $5.84 per RU/s x 100 ├─ Transactional storage (East US) Monthly cost depends on usage: $0.25 per GB ├─ Periodic backup (East US) Monthly cost depends on usage: $0.12 per GB └─ Restored data Monthly cost depends on usage: $0.15 per GB diff --git a/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_keyspace_test_with_blank_geo_location/cosmosdb_cassandra_keyspace_test_with_blank_geo_location.tf b/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_keyspace_test_with_blank_geo_location/cosmosdb_cassandra_keyspace_test_with_blank_geo_location.tf index 25f3e81039f..6ae70617ea4 100644 --- a/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_keyspace_test_with_blank_geo_location/cosmosdb_cassandra_keyspace_test_with_blank_geo_location.tf +++ b/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_keyspace_test_with_blank_geo_location/cosmosdb_cassandra_keyspace_test_with_blank_geo_location.tf @@ -9,11 +9,11 @@ resource "azurerm_resource_group" "example" { } resource "azurerm_cosmosdb_account" "with_blank_geo_location" { - name = "tfex-cosmosdb-account" - resource_group_name = azurerm_resource_group.example.name - location = azurerm_resource_group.example.location - offer_type = "Standard" - enable_multiple_write_locations = true + name = "tfex-cosmosdb-account" + resource_group_name = azurerm_resource_group.example.name + location = azurerm_resource_group.example.location + offer_type = "Standard" + consistency_policy { consistency_level = "Strong" diff --git a/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_table_test/cosmosdb_cassandra_table_test.golden b/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_table_test/cosmosdb_cassandra_table_test.golden index 01048b17f4c..368414827bb 100644 --- a/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_table_test/cosmosdb_cassandra_table_test.golden +++ b/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_table_test/cosmosdb_cassandra_table_test.golden @@ -29,15 +29,15 @@ └─ Restored data 3,000 GB $450.00 azurerm_cosmosdb_cassandra_table.mutli-master_backup2copies - ├─ Provisioned throughput (provisioned, West US) 10 RU/s x 100 $116.80 - ├─ Provisioned throughput (provisioned, Central US) 10 RU/s x 100 $116.80 + ├─ Provisioned throughput (provisioned, West US) 10 RU/s x 100 $58.40 + ├─ Provisioned throughput (provisioned, Central US) 10 RU/s x 100 $58.40 ├─ Transactional storage (West US) 1,000 GB $250.00 ├─ Transactional storage (Central US) 1,000 GB $250.00 └─ Restored data Monthly cost depends on usage: $0.15 per GB azurerm_cosmosdb_cassandra_keyspace.mutli-master_backup2copies - ├─ Provisioned throughput (provisioned, West US) 10 RU/s x 100 $116.80 - ├─ Provisioned throughput (provisioned, Central US) 10 RU/s x 100 $116.80 + ├─ Provisioned throughput (provisioned, West US) 10 RU/s x 100 $58.40 + ├─ Provisioned throughput (provisioned, Central US) 10 RU/s x 100 $58.40 ├─ Transactional storage (West US) Monthly cost depends on usage: $0.25 per GB ├─ Transactional storage (Central US) Monthly cost depends on usage: $0.25 per GB ├─ Periodic backup (West US) Monthly cost depends on usage: $0.15 per GB @@ -89,7 +89,7 @@ ├─ Periodic backup (West US) Monthly cost depends on usage: $0.15 per GB └─ Restored data Monthly cost depends on usage: $0.15 per GB - OVERALL TOTAL $5,455.03 + OVERALL TOTAL $5,221.43 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -104,7 +104,7 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $5,455 ┃ - ┃ $5,455 ┃ +┃ main ┃ $5,221 ┃ - ┃ $5,221 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ Logs: diff --git a/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_table_test/cosmosdb_cassandra_table_test.tf b/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_table_test/cosmosdb_cassandra_table_test.tf index 1ba3efd6bf6..fc074ee710f 100644 --- a/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_table_test/cosmosdb_cassandra_table_test.tf +++ b/internal/providers/terraform/azure/testdata/cosmosdb_cassandra_table_test/cosmosdb_cassandra_table_test.tf @@ -57,11 +57,11 @@ resource "azurerm_cosmosdb_account" "continuous_backup" { } resource "azurerm_cosmosdb_account" "multi-master_backup2copies" { - name = "tfex-cosmosdb-account" - resource_group_name = azurerm_resource_group.example.name - location = azurerm_resource_group.example.location - offer_type = "Standard" - enable_multiple_write_locations = true + name = "tfex-cosmosdb-account" + resource_group_name = azurerm_resource_group.example.name + location = azurerm_resource_group.example.location + offer_type = "Standard" + consistency_policy { consistency_level = "Strong" diff --git a/internal/providers/terraform/azure/testdata/cosmosdb_gremlin_database_test/cosmosdb_gremlin_database_test.golden b/internal/providers/terraform/azure/testdata/cosmosdb_gremlin_database_test/cosmosdb_gremlin_database_test.golden index 1aa45626d87..fa8282a4eb2 100644 --- a/internal/providers/terraform/azure/testdata/cosmosdb_gremlin_database_test/cosmosdb_gremlin_database_test.golden +++ b/internal/providers/terraform/azure/testdata/cosmosdb_gremlin_database_test/cosmosdb_gremlin_database_test.golden @@ -29,8 +29,8 @@ └─ Restored data 3,000 GB $450.00 azurerm_cosmosdb_gremlin_database.mutli-master_backup2copies - ├─ Provisioned throughput (provisioned, West US) 10 RU/s x 100 $116.80 - ├─ Provisioned throughput (provisioned, Central US) 10 RU/s x 100 $116.80 + ├─ Provisioned throughput (provisioned, West US) 10 RU/s x 100 $58.40 + ├─ Provisioned throughput (provisioned, Central US) 10 RU/s x 100 $58.40 ├─ Transactional storage (West US) 1,000 GB $250.00 ├─ Transactional storage (Central US) 1,000 GB $250.00 └─ Restored data Monthly cost depends on usage: $0.15 per GB @@ -44,7 +44,7 @@ ├─ Periodic backup (West US) Monthly cost depends on usage: $0.15 per GB └─ Restored data Monthly cost depends on usage: $0.15 per GB - OVERALL TOTAL $5,155.73 + OVERALL TOTAL $5,038.93 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -56,5 +56,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $5,156 ┃ - ┃ $5,156 ┃ +┃ main ┃ $5,039 ┃ - ┃ $5,039 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/cosmosdb_gremlin_database_test/cosmosdb_gremlin_database_test.tf b/internal/providers/terraform/azure/testdata/cosmosdb_gremlin_database_test/cosmosdb_gremlin_database_test.tf index cbc146a2bad..7bd5baa1cfb 100644 --- a/internal/providers/terraform/azure/testdata/cosmosdb_gremlin_database_test/cosmosdb_gremlin_database_test.tf +++ b/internal/providers/terraform/azure/testdata/cosmosdb_gremlin_database_test/cosmosdb_gremlin_database_test.tf @@ -57,11 +57,11 @@ resource "azurerm_cosmosdb_account" "continuous_backup" { } resource "azurerm_cosmosdb_account" "multi-master_backup2copies" { - name = "tfex-cosmosdb-account" - resource_group_name = azurerm_resource_group.example.name - location = azurerm_resource_group.example.location - offer_type = "Standard" - enable_multiple_write_locations = true + name = "tfex-cosmosdb-account" + resource_group_name = azurerm_resource_group.example.name + location = azurerm_resource_group.example.location + offer_type = "Standard" + consistency_policy { consistency_level = "Strong" diff --git a/internal/providers/terraform/azure/testdata/cosmosdb_gremlin_graph_test/cosmosdb_gremlin_graph_test.golden b/internal/providers/terraform/azure/testdata/cosmosdb_gremlin_graph_test/cosmosdb_gremlin_graph_test.golden index 92b5cdc560b..b09569877d8 100644 --- a/internal/providers/terraform/azure/testdata/cosmosdb_gremlin_graph_test/cosmosdb_gremlin_graph_test.golden +++ b/internal/providers/terraform/azure/testdata/cosmosdb_gremlin_graph_test/cosmosdb_gremlin_graph_test.golden @@ -29,8 +29,8 @@ └─ Restored data 3,000 GB $450.00 azurerm_cosmosdb_gremlin_graph.mutli-master_backup2copies - ├─ Provisioned throughput (provisioned, West US) 10 RU/s x 100 $116.80 - ├─ Provisioned throughput (provisioned, Central US) 10 RU/s x 100 $116.80 + ├─ Provisioned throughput (provisioned, West US) 10 RU/s x 100 $58.40 + ├─ Provisioned throughput (provisioned, Central US) 10 RU/s x 100 $58.40 ├─ Transactional storage (West US) 1,000 GB $250.00 ├─ Transactional storage (Central US) 1,000 GB $250.00 └─ Restored data Monthly cost depends on usage: $0.15 per GB @@ -53,7 +53,7 @@ ├─ Periodic backup (West US) Monthly cost depends on usage: $0.15 per GB └─ Restored data Monthly cost depends on usage: $0.15 per GB - OVERALL TOTAL $5,155.73 + OVERALL TOTAL $5,038.93 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -65,5 +65,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $5,156 ┃ - ┃ $5,156 ┃ +┃ main ┃ $5,039 ┃ - ┃ $5,039 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/cosmosdb_gremlin_graph_test/cosmosdb_gremlin_graph_test.tf b/internal/providers/terraform/azure/testdata/cosmosdb_gremlin_graph_test/cosmosdb_gremlin_graph_test.tf index 21695b09711..4bf848ddf4b 100644 --- a/internal/providers/terraform/azure/testdata/cosmosdb_gremlin_graph_test/cosmosdb_gremlin_graph_test.tf +++ b/internal/providers/terraform/azure/testdata/cosmosdb_gremlin_graph_test/cosmosdb_gremlin_graph_test.tf @@ -57,11 +57,11 @@ resource "azurerm_cosmosdb_account" "continuous_backup" { } resource "azurerm_cosmosdb_account" "multi-master_backup2copies" { - name = "tfex-cosmosdb-account" - resource_group_name = azurerm_resource_group.example.name - location = azurerm_resource_group.example.location - offer_type = "Standard" - enable_multiple_write_locations = true + name = "tfex-cosmosdb-account" + resource_group_name = azurerm_resource_group.example.name + location = azurerm_resource_group.example.location + offer_type = "Standard" + consistency_policy { consistency_level = "Strong" diff --git a/internal/providers/terraform/azure/testdata/cosmosdb_mongo_collection_test/cosmosdb_mongo_collection_test.golden b/internal/providers/terraform/azure/testdata/cosmosdb_mongo_collection_test/cosmosdb_mongo_collection_test.golden index 33c8e8eeeb8..a6b7730bf13 100644 --- a/internal/providers/terraform/azure/testdata/cosmosdb_mongo_collection_test/cosmosdb_mongo_collection_test.golden +++ b/internal/providers/terraform/azure/testdata/cosmosdb_mongo_collection_test/cosmosdb_mongo_collection_test.golden @@ -29,15 +29,15 @@ └─ Restored data 3,000 GB $450.00 azurerm_cosmosdb_mongo_collection.mutli-master_backup2copies - ├─ Provisioned throughput (provisioned, West US) 10 RU/s x 100 $116.80 - ├─ Provisioned throughput (provisioned, Central US) 10 RU/s x 100 $116.80 + ├─ Provisioned throughput (provisioned, West US) 10 RU/s x 100 $58.40 + ├─ Provisioned throughput (provisioned, Central US) 10 RU/s x 100 $58.40 ├─ Transactional storage (West US) 1,000 GB $250.00 ├─ Transactional storage (Central US) 1,000 GB $250.00 └─ Restored data Monthly cost depends on usage: $0.15 per GB azurerm_cosmosdb_mongo_database.mutli-master_backup2copies - ├─ Provisioned throughput (provisioned, West US) 10 RU/s x 100 $116.80 - ├─ Provisioned throughput (provisioned, Central US) 10 RU/s x 100 $116.80 + ├─ Provisioned throughput (provisioned, West US) 10 RU/s x 100 $58.40 + ├─ Provisioned throughput (provisioned, Central US) 10 RU/s x 100 $58.40 ├─ Transactional storage (West US) Monthly cost depends on usage: $0.25 per GB ├─ Transactional storage (Central US) Monthly cost depends on usage: $0.25 per GB ├─ Periodic backup (West US) Monthly cost depends on usage: $0.15 per GB @@ -89,7 +89,7 @@ ├─ Periodic backup (West US) Monthly cost depends on usage: $0.15 per GB └─ Restored data Monthly cost depends on usage: $0.15 per GB - OVERALL TOTAL $5,455.03 + OVERALL TOTAL $5,221.43 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -101,5 +101,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $5,455 ┃ - ┃ $5,455 ┃ +┃ main ┃ $5,221 ┃ - ┃ $5,221 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/cosmosdb_mongo_collection_test/cosmosdb_mongo_collection_test.tf b/internal/providers/terraform/azure/testdata/cosmosdb_mongo_collection_test/cosmosdb_mongo_collection_test.tf index ef744399a5d..4cffcf4a8f9 100644 --- a/internal/providers/terraform/azure/testdata/cosmosdb_mongo_collection_test/cosmosdb_mongo_collection_test.tf +++ b/internal/providers/terraform/azure/testdata/cosmosdb_mongo_collection_test/cosmosdb_mongo_collection_test.tf @@ -57,11 +57,11 @@ resource "azurerm_cosmosdb_account" "continuous_backup" { } resource "azurerm_cosmosdb_account" "multi-master_backup2copies" { - name = "tfex-cosmosdb-account" - resource_group_name = azurerm_resource_group.example.name - location = azurerm_resource_group.example.location - offer_type = "Standard" - enable_multiple_write_locations = true + name = "tfex-cosmosdb-account" + resource_group_name = azurerm_resource_group.example.name + location = azurerm_resource_group.example.location + offer_type = "Standard" + consistency_policy { consistency_level = "Strong" diff --git a/internal/providers/terraform/azure/testdata/cosmosdb_mongo_database_test/cosmosdb_mongo_database_test.golden b/internal/providers/terraform/azure/testdata/cosmosdb_mongo_database_test/cosmosdb_mongo_database_test.golden index 494d2aed0d9..35c54f17570 100644 --- a/internal/providers/terraform/azure/testdata/cosmosdb_mongo_database_test/cosmosdb_mongo_database_test.golden +++ b/internal/providers/terraform/azure/testdata/cosmosdb_mongo_database_test/cosmosdb_mongo_database_test.golden @@ -29,8 +29,8 @@ └─ Restored data 3,000 GB $450.00 azurerm_cosmosdb_mongo_database.mutli-master_backup2copies - ├─ Provisioned throughput (provisioned, West US) 10 RU/s x 100 $116.80 - ├─ Provisioned throughput (provisioned, Central US) 10 RU/s x 100 $116.80 + ├─ Provisioned throughput (provisioned, West US) 10 RU/s x 100 $58.40 + ├─ Provisioned throughput (provisioned, Central US) 10 RU/s x 100 $58.40 ├─ Transactional storage (West US) 1,000 GB $250.00 ├─ Transactional storage (Central US) 1,000 GB $250.00 └─ Restored data Monthly cost depends on usage: $0.15 per GB @@ -44,7 +44,7 @@ ├─ Periodic backup (West US) Monthly cost depends on usage: $0.15 per GB └─ Restored data Monthly cost depends on usage: $0.15 per GB - OVERALL TOTAL $5,155.73 + OVERALL TOTAL $5,038.93 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -56,5 +56,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $5,156 ┃ - ┃ $5,156 ┃ +┃ main ┃ $5,039 ┃ - ┃ $5,039 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/cosmosdb_mongo_database_test/cosmosdb_mongo_database_test.tf b/internal/providers/terraform/azure/testdata/cosmosdb_mongo_database_test/cosmosdb_mongo_database_test.tf index 6afcb4a70f2..e01c046a12b 100644 --- a/internal/providers/terraform/azure/testdata/cosmosdb_mongo_database_test/cosmosdb_mongo_database_test.tf +++ b/internal/providers/terraform/azure/testdata/cosmosdb_mongo_database_test/cosmosdb_mongo_database_test.tf @@ -57,11 +57,11 @@ resource "azurerm_cosmosdb_account" "continuous_backup" { } resource "azurerm_cosmosdb_account" "multi-master_backup2copies" { - name = "tfex-cosmosdb-account" - resource_group_name = azurerm_resource_group.example.name - location = azurerm_resource_group.example.location - offer_type = "Standard" - enable_multiple_write_locations = true + name = "tfex-cosmosdb-account" + resource_group_name = azurerm_resource_group.example.name + location = azurerm_resource_group.example.location + offer_type = "Standard" + consistency_policy { consistency_level = "Strong" diff --git a/internal/providers/terraform/azure/testdata/cosmosdb_sql_container_test/cosmosdb_sql_container_test.tf b/internal/providers/terraform/azure/testdata/cosmosdb_sql_container_test/cosmosdb_sql_container_test.tf index 6ae2460e89f..4677453576c 100644 --- a/internal/providers/terraform/azure/testdata/cosmosdb_sql_container_test/cosmosdb_sql_container_test.tf +++ b/internal/providers/terraform/azure/testdata/cosmosdb_sql_container_test/cosmosdb_sql_container_test.tf @@ -57,11 +57,11 @@ resource "azurerm_cosmosdb_account" "continuous_backup" { } resource "azurerm_cosmosdb_account" "multi-master_backup2copies" { - name = "tfex-cosmosdb-account" - resource_group_name = azurerm_resource_group.example.name - location = azurerm_resource_group.example.location - offer_type = "Standard" - enable_multiple_write_locations = true + name = "tfex-cosmosdb-account" + resource_group_name = azurerm_resource_group.example.name + location = azurerm_resource_group.example.location + offer_type = "Standard" + consistency_policy { consistency_level = "Strong" @@ -95,7 +95,7 @@ resource "azurerm_cosmosdb_sql_container" "non-usage_autoscale" { resource_group_name = azurerm_cosmosdb_account.example.resource_group_name account_name = azurerm_cosmosdb_account.example.name database_name = azurerm_cosmosdb_sql_database.example.name - partition_key_path = "/definition/id" + partition_key_paths = ["/definition/id"] autoscale_settings { max_throughput = 4000 } @@ -106,7 +106,7 @@ resource "azurerm_cosmosdb_sql_container" "autoscale" { resource_group_name = azurerm_cosmosdb_account.example.resource_group_name account_name = azurerm_cosmosdb_account.continuous_backup.name database_name = azurerm_cosmosdb_sql_database.example.name - partition_key_path = "/definition/id" + partition_key_paths = ["/definition/id"] autoscale_settings { max_throughput = 6000 } @@ -117,7 +117,7 @@ resource "azurerm_cosmosdb_sql_container" "provisioned" { resource_group_name = azurerm_cosmosdb_account.example.resource_group_name account_name = azurerm_cosmosdb_account.continuous_backup.name database_name = azurerm_cosmosdb_sql_database.example.name - partition_key_path = "/definition/id" + partition_key_paths = ["/definition/id"] throughput = 500 } @@ -126,7 +126,7 @@ resource "azurerm_cosmosdb_sql_container" "mutli-master_backup2copies" { resource_group_name = azurerm_cosmosdb_account.example.resource_group_name account_name = azurerm_cosmosdb_account.example.name database_name = azurerm_cosmosdb_sql_database.example.name - partition_key_path = "/definition/id" + partition_key_paths = ["/definition/id"] throughput = 1000 } @@ -135,5 +135,5 @@ resource "azurerm_cosmosdb_sql_container" "serverless" { resource_group_name = azurerm_cosmosdb_account.example.resource_group_name account_name = azurerm_cosmosdb_account.example.name database_name = azurerm_cosmosdb_sql_database.example.name - partition_key_path = "/definition/id" + partition_key_paths = ["/definition/id"] } diff --git a/internal/providers/terraform/azure/testdata/cosmosdb_sql_database_test/cosmosdb_sql_database_test.golden b/internal/providers/terraform/azure/testdata/cosmosdb_sql_database_test/cosmosdb_sql_database_test.golden index eee2c81cce3..c1645dbafad 100644 --- a/internal/providers/terraform/azure/testdata/cosmosdb_sql_database_test/cosmosdb_sql_database_test.golden +++ b/internal/providers/terraform/azure/testdata/cosmosdb_sql_database_test/cosmosdb_sql_database_test.golden @@ -29,8 +29,8 @@ └─ Restored data 3,000 GB $450.00 azurerm_cosmosdb_sql_database.mutli-master_backup2copies - ├─ Provisioned throughput (provisioned, West US) 10 RU/s x 100 $116.80 - ├─ Provisioned throughput (provisioned, Central US) 10 RU/s x 100 $116.80 + ├─ Provisioned throughput (provisioned, West US) 10 RU/s x 100 $58.40 + ├─ Provisioned throughput (provisioned, Central US) 10 RU/s x 100 $58.40 ├─ Transactional storage (West US) 1,000 GB $250.00 ├─ Transactional storage (Central US) 1,000 GB $250.00 └─ Restored data Monthly cost depends on usage: $0.15 per GB @@ -53,7 +53,7 @@ ├─ Periodic backup (West US) Monthly cost depends on usage: $0.15 per GB └─ Restored data Monthly cost depends on usage: $0.15 per GB - OVERALL TOTAL $5,184.93 + OVERALL TOTAL $5,068.13 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -65,5 +65,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $5,185 ┃ - ┃ $5,185 ┃ +┃ main ┃ $5,068 ┃ - ┃ $5,068 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/cosmosdb_sql_database_test/cosmosdb_sql_database_test.tf b/internal/providers/terraform/azure/testdata/cosmosdb_sql_database_test/cosmosdb_sql_database_test.tf index 24ec81a80c4..8209614a456 100644 --- a/internal/providers/terraform/azure/testdata/cosmosdb_sql_database_test/cosmosdb_sql_database_test.tf +++ b/internal/providers/terraform/azure/testdata/cosmosdb_sql_database_test/cosmosdb_sql_database_test.tf @@ -57,11 +57,11 @@ resource "azurerm_cosmosdb_account" "continuous_backup" { } resource "azurerm_cosmosdb_account" "multi-master_backup2copies" { - name = "tfex-cosmosdb-account-backup2copies" - resource_group_name = azurerm_resource_group.example.name - location = azurerm_resource_group.example.location - offer_type = "Standard" - enable_multiple_write_locations = true + name = "tfex-cosmosdb-account-backup2copies" + resource_group_name = azurerm_resource_group.example.name + location = azurerm_resource_group.example.location + offer_type = "Standard" + consistency_policy { consistency_level = "Strong" diff --git a/internal/providers/terraform/azure/testdata/cosmosdb_table_test/cosmosdb_table_test.golden b/internal/providers/terraform/azure/testdata/cosmosdb_table_test/cosmosdb_table_test.golden index bab8a89a605..1d3b736fa46 100644 --- a/internal/providers/terraform/azure/testdata/cosmosdb_table_test/cosmosdb_table_test.golden +++ b/internal/providers/terraform/azure/testdata/cosmosdb_table_test/cosmosdb_table_test.golden @@ -29,8 +29,8 @@ └─ Restored data 3,000 GB $450.00 azurerm_cosmosdb_table.mutli-master_backup2copies - ├─ Provisioned throughput (provisioned, West US) 10 RU/s x 100 $116.80 - ├─ Provisioned throughput (provisioned, Central US) 10 RU/s x 100 $116.80 + ├─ Provisioned throughput (provisioned, West US) 10 RU/s x 100 $58.40 + ├─ Provisioned throughput (provisioned, Central US) 10 RU/s x 100 $58.40 ├─ Transactional storage (West US) 1,000 GB $250.00 ├─ Transactional storage (Central US) 1,000 GB $250.00 └─ Restored data Monthly cost depends on usage: $0.15 per GB @@ -44,7 +44,7 @@ ├─ Periodic backup (West US) Monthly cost depends on usage: $0.15 per GB └─ Restored data Monthly cost depends on usage: $0.15 per GB - OVERALL TOTAL $5,155.73 + OVERALL TOTAL $5,038.93 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -58,7 +58,7 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $5,156 ┃ - ┃ $5,156 ┃ +┃ main ┃ $5,039 ┃ - ┃ $5,039 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ Logs: diff --git a/internal/providers/terraform/azure/testdata/cosmosdb_table_test/cosmosdb_table_test.tf b/internal/providers/terraform/azure/testdata/cosmosdb_table_test/cosmosdb_table_test.tf index 25d220582d4..884e3538cde 100644 --- a/internal/providers/terraform/azure/testdata/cosmosdb_table_test/cosmosdb_table_test.tf +++ b/internal/providers/terraform/azure/testdata/cosmosdb_table_test/cosmosdb_table_test.tf @@ -57,11 +57,11 @@ resource "azurerm_cosmosdb_account" "continuous_backup" { } resource "azurerm_cosmosdb_account" "multi-master_backup2copies" { - name = "tfex-cosmosdb-account" - resource_group_name = azurerm_resource_group.example.name - location = azurerm_resource_group.example.location - offer_type = "Standard" - enable_multiple_write_locations = true + name = "tfex-cosmosdb-account" + resource_group_name = azurerm_resource_group.example.name + location = azurerm_resource_group.example.location + offer_type = "Standard" + consistency_policy { consistency_level = "Strong" diff --git a/internal/providers/terraform/azure/testdata/event_hubs_namespace_test/event_hubs_namespace_test.tf b/internal/providers/terraform/azure/testdata/event_hubs_namespace_test/event_hubs_namespace_test.tf index 0fa7ebc6487..5ce15213759 100644 --- a/internal/providers/terraform/azure/testdata/event_hubs_namespace_test/event_hubs_namespace_test.tf +++ b/internal/providers/terraform/azure/testdata/event_hubs_namespace_test/event_hubs_namespace_test.tf @@ -43,8 +43,8 @@ resource "azurerm_eventhub_namespace" "premium" { location = azurerm_resource_group.example.location resource_group_name = azurerm_resource_group.example.name sku = "Premium" - zone_redundant = true - capacity = 8 + + capacity = 8 } resource "azurerm_eventhub_namespace" "premiumWithoutUsage" { @@ -52,7 +52,7 @@ resource "azurerm_eventhub_namespace" "premiumWithoutUsage" { location = azurerm_resource_group.example.location resource_group_name = azurerm_resource_group.example.name sku = "Premium" - zone_redundant = true + } resource "azurerm_eventhub_namespace" "dedicated" { diff --git a/internal/providers/terraform/azure/testdata/kubernetes_cluster_test/kubernetes_cluster_test.golden b/internal/providers/terraform/azure/testdata/kubernetes_cluster_test/kubernetes_cluster_test.golden index 0632756e9b5..4d9fba070b4 100644 --- a/internal/providers/terraform/azure/testdata/kubernetes_cluster_test/kubernetes_cluster_test.golden +++ b/internal/providers/terraform/azure/testdata/kubernetes_cluster_test/kubernetes_cluster_test.golden @@ -4,7 +4,7 @@ azurerm_kubernetes_cluster.windows ├─ Uptime SLA 730 hours $73.00 └─ default_node_pool - ├─ Instance usage (Windows, pay as you go, Standard_D2_v2) 3,650 hours $919.80 + ├─ Instance usage (Windows, pay as you go, Standard_D2_v2) 3,650 hours $868.70 └─ os_disk └─ Storage (S4, LRS) 5 months $7.68 @@ -44,7 +44,7 @@ └─ os_disk └─ Storage (S10, LRS) 1 months $5.89 - OVERALL TOTAL $2,835.67 + OVERALL TOTAL $2,784.57 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -56,5 +56,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $2,835 ┃ $0.50 ┃ $2,836 ┃ +┃ main ┃ $2,784 ┃ $0.50 ┃ $2,785 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/lb_outbound_rule_test/lb_outbound_rule_test.golden b/internal/providers/terraform/azure/testdata/lb_outbound_rule_test/lb_outbound_rule_test.golden index e364f32540e..07561666d87 100644 --- a/internal/providers/terraform/azure/testdata/lb_outbound_rule_test/lb_outbound_rule_test.golden +++ b/internal/providers/terraform/azure/testdata/lb_outbound_rule_test/lb_outbound_rule_test.golden @@ -5,12 +5,12 @@ └─ Rule usage 730 hours $7.30 azurerm_public_ip.example - └─ IP address (static, regional) 730 hours $2.63 + └─ IP address (static, regional) 730 hours $3.65 azurerm_lb.example └─ Data processed Monthly cost depends on usage: $0.005 per GB - OVERALL TOTAL $9.93 + OVERALL TOTAL $10.95 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -22,5 +22,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $10 ┃ - ┃ $10 ┃ +┃ main ┃ $11 ┃ - ┃ $11 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/lb_outbound_rule_v2_test/lb_outbound_rule_v2_test.golden b/internal/providers/terraform/azure/testdata/lb_outbound_rule_v2_test/lb_outbound_rule_v2_test.golden index e364f32540e..07561666d87 100644 --- a/internal/providers/terraform/azure/testdata/lb_outbound_rule_v2_test/lb_outbound_rule_v2_test.golden +++ b/internal/providers/terraform/azure/testdata/lb_outbound_rule_v2_test/lb_outbound_rule_v2_test.golden @@ -5,12 +5,12 @@ └─ Rule usage 730 hours $7.30 azurerm_public_ip.example - └─ IP address (static, regional) 730 hours $2.63 + └─ IP address (static, regional) 730 hours $3.65 azurerm_lb.example └─ Data processed Monthly cost depends on usage: $0.005 per GB - OVERALL TOTAL $9.93 + OVERALL TOTAL $10.95 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -22,5 +22,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $10 ┃ - ┃ $10 ┃ +┃ main ┃ $11 ┃ - ┃ $11 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/lb_rule_test/lb_rule_test.golden b/internal/providers/terraform/azure/testdata/lb_rule_test/lb_rule_test.golden index 4665ee7944f..96ee0d08d06 100644 --- a/internal/providers/terraform/azure/testdata/lb_rule_test/lb_rule_test.golden +++ b/internal/providers/terraform/azure/testdata/lb_rule_test/lb_rule_test.golden @@ -5,18 +5,18 @@ └─ Rule usage 730 hours $7.30 azurerm_public_ip.example_withBasicSku - └─ IP address (static, regional) 730 hours $2.63 + └─ IP address (static, regional) 730 hours $3.65 azurerm_public_ip.example_withDefaultSku - └─ IP address (static, regional) 730 hours $2.63 + └─ IP address (static, regional) 730 hours $3.65 azurerm_public_ip.example_withStandardSku - └─ IP address (static, regional) 730 hours $2.63 + └─ IP address (static, regional) 730 hours $3.65 azurerm_lb.example_withStandardSku └─ Data processed Monthly cost depends on usage: $0.005 per GB - OVERALL TOTAL $15.18 + OVERALL TOTAL $18.25 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -28,5 +28,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $15 ┃ - ┃ $15 ┃ +┃ main ┃ $18 ┃ - ┃ $18 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/lb_rule_v2_test/lb_rule_v2_test.golden b/internal/providers/terraform/azure/testdata/lb_rule_v2_test/lb_rule_v2_test.golden index 4665ee7944f..08f3ba03b4b 100644 --- a/internal/providers/terraform/azure/testdata/lb_rule_v2_test/lb_rule_v2_test.golden +++ b/internal/providers/terraform/azure/testdata/lb_rule_v2_test/lb_rule_v2_test.golden @@ -4,19 +4,19 @@ azurerm_lb_rule.rules_withStandardSku └─ Rule usage 730 hours $7.30 - azurerm_public_ip.example_withBasicSku - └─ IP address (static, regional) 730 hours $2.63 - azurerm_public_ip.example_withDefaultSku - └─ IP address (static, regional) 730 hours $2.63 + └─ IP address (static, regional) 730 hours $3.65 azurerm_public_ip.example_withStandardSku + └─ IP address (static, regional) 730 hours $3.65 + + azurerm_public_ip.example_withBasicSku └─ IP address (static, regional) 730 hours $2.63 azurerm_lb.example_withStandardSku └─ Data processed Monthly cost depends on usage: $0.005 per GB - OVERALL TOTAL $15.18 + OVERALL TOTAL $17.23 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -28,5 +28,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $15 ┃ - ┃ $15 ┃ +┃ main ┃ $17 ┃ - ┃ $17 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/lb_rule_v2_test/lb_rule_v2_test.tf b/internal/providers/terraform/azure/testdata/lb_rule_v2_test/lb_rule_v2_test.tf index 2b64350008d..1ef49f8980d 100644 --- a/internal/providers/terraform/azure/testdata/lb_rule_v2_test/lb_rule_v2_test.tf +++ b/internal/providers/terraform/azure/testdata/lb_rule_v2_test/lb_rule_v2_test.tf @@ -25,6 +25,7 @@ resource "azurerm_public_ip" "example_withDefaultSku" { location = "West US" resource_group_name = azurerm_resource_group.example.name allocation_method = "Static" + sku = "Standard" } resource "azurerm_lb" "example_withDefaultSku" { @@ -53,6 +54,7 @@ resource "azurerm_public_ip" "example_withBasicSku" { location = "West US" resource_group_name = azurerm_resource_group.example.name allocation_method = "Static" + sku = "Basic" } resource "azurerm_lb" "example_withBasicSku" { @@ -82,6 +84,7 @@ resource "azurerm_public_ip" "example_withStandardSku" { location = "West US" resource_group_name = azurerm_resource_group.example.name allocation_method = "Static" + sku = "Standard" } resource "azurerm_lb" "example_withStandardSku" { diff --git a/internal/providers/terraform/azure/testdata/log_analytics_workspace_test/log_analytics_workspace_test.golden b/internal/providers/terraform/azure/testdata/log_analytics_workspace_test/log_analytics_workspace_test.golden index 7f77774d830..f58c45615d5 100644 --- a/internal/providers/terraform/azure/testdata/log_analytics_workspace_test/log_analytics_workspace_test.golden +++ b/internal/providers/terraform/azure/testdata/log_analytics_workspace_test/log_analytics_workspace_test.golden @@ -118,9 +118,9 @@ *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. ────────────────────────────────── -22 cloud resources were detected: +21 cloud resources were detected: ∙ 12 were estimated -∙ 6 were free +∙ 5 were free ∙ 4 are not supported yet, see https://infracost.io/requested-resources: ∙ 4 x azurerm_log_analytics_workspace diff --git a/internal/providers/terraform/azure/testdata/log_analytics_workspace_test/log_analytics_workspace_test.tf b/internal/providers/terraform/azure/testdata/log_analytics_workspace_test/log_analytics_workspace_test.tf index 8c97b7aeaad..296e4a37749 100644 --- a/internal/providers/terraform/azure/testdata/log_analytics_workspace_test/log_analytics_workspace_test.tf +++ b/internal/providers/terraform/azure/testdata/log_analytics_workspace_test/log_analytics_workspace_test.tf @@ -8,13 +8,6 @@ resource "azurerm_resource_group" "example" { location = "West Europe" } -resource "azurerm_log_analytics_workspace" "free_workspace" { - name = "acctest-free" - location = azurerm_resource_group.example.location - resource_group_name = azurerm_resource_group.example.name - sku = "Free" -} - resource "azurerm_log_analytics_workspace" "per_gb_data_ingestion" { name = "acctest-01" location = azurerm_resource_group.example.location diff --git a/internal/providers/terraform/azure/testdata/machine_learning_compute_instance_test/machine_learning_compute_instance_test.golden b/internal/providers/terraform/azure/testdata/machine_learning_compute_instance_test/machine_learning_compute_instance_test.golden index 06eb89b201a..7669527acb3 100644 --- a/internal/providers/terraform/azure/testdata/machine_learning_compute_instance_test/machine_learning_compute_instance_test.golden +++ b/internal/providers/terraform/azure/testdata/machine_learning_compute_instance_test/machine_learning_compute_instance_test.golden @@ -2,58 +2,58 @@ Name Monthly Qty Unit Monthly Cost azurerm_machine_learning_compute_instance.example["Standard_NV24"] - └─ Instance usage (Linux, pay as you go, Standard_NV24) 730 hours $3,985.80 + └─ Instance usage (Linux, pay as you go, Standard_NV24) 730 hours $3,328.80 azurerm_machine_learning_compute_instance.example["Standard_NC24"] - └─ Instance usage (Linux, pay as you go, Standard_NC24) 730 hours $3,406.18 + └─ Instance usage (Linux, pay as you go, Standard_NC24) 730 hours $2,628.00 azurerm_machine_learning_compute_instance.example["Standard_NV12"] - └─ Instance usage (Linux, pay as you go, Standard_NV12) 730 hours $1,992.90 + └─ Instance usage (Linux, pay as you go, Standard_NV12) 730 hours $1,664.40 azurerm_machine_learning_compute_instance.example["Standard_NC12"] - └─ Instance usage (Linux, pay as you go, Standard_NC12) 730 hours $1,703.09 + └─ Instance usage (Linux, pay as you go, Standard_NC12) 730 hours $1,314.00 azurerm_machine_learning_compute_instance.example["Standard_NV6"] - └─ Instance usage (Linux, pay as you go, Standard_NV6) 730 hours $996.45 + └─ Instance usage (Linux, pay as you go, Standard_NV6) 730 hours $832.20 azurerm_machine_learning_compute_instance.example["Standard_E16s_v3"] - └─ Instance usage (Linux, pay as you go, Standard_E16s_v3) 730 hours $934.40 + └─ Instance usage (Linux, pay as you go, Standard_E16s_v3) 730 hours $735.84 azurerm_machine_learning_compute_instance.example["Standard_NC6"] - └─ Instance usage (Linux, pay as you go, Standard_NC6) 730 hours $851.18 + └─ Instance usage (Linux, pay as you go, Standard_NC6) 730 hours $657.00 azurerm_machine_learning_compute_instance.example["Standard_D16s_v3"] - └─ Instance usage (Linux, pay as you go, Standard_D16s_v3) 730 hours $700.80 + └─ Instance usage (Linux, pay as you go, Standard_D16s_v3) 730 hours $560.64 azurerm_machine_learning_compute_instance.example["Standard_F16s_v2"] - └─ Instance usage (Linux, pay as you go, Standard_F16s_v2) 730 hours $566.48 + └─ Instance usage (Linux, pay as you go, Standard_F16s_v2) 730 hours $494.21 azurerm_machine_learning_compute_instance.example["Standard_E8s_v3"] - └─ Instance usage (Linux, pay as you go, Standard_E8s_v3) 730 hours $467.20 + └─ Instance usage (Linux, pay as you go, Standard_E8s_v3) 730 hours $367.92 azurerm_machine_learning_compute_instance.example["Standard_D8s_v3"] - └─ Instance usage (Linux, pay as you go, Standard_D8s_v3) 730 hours $350.40 - - azurerm_machine_learning_compute_instance.example["Standard_F8s_v2"] - └─ Instance usage (Linux, pay as you go, Standard_F8s_v2) 730 hours $283.24 + └─ Instance usage (Linux, pay as you go, Standard_D8s_v3) 730 hours $280.32 azurerm_machine_learning_compute_instance.example["Standard_DS12_v2"] - └─ Instance usage (Linux, pay as you go, Standard_DS12_v2) 730 hours $276.67 + └─ Instance usage (Linux, pay as you go, Standard_DS12_v2) 730 hours $270.83 + + azurerm_machine_learning_compute_instance.example["Standard_F8s_v2"] + └─ Instance usage (Linux, pay as you go, Standard_F8s_v2) 730 hours $246.74 azurerm_machine_learning_compute_instance.example["Standard_E4s_v3"] - └─ Instance usage (Linux, pay as you go, Standard_E4s_v3) 730 hours $233.60 + └─ Instance usage (Linux, pay as you go, Standard_E4s_v3) 730 hours $183.96 azurerm_machine_learning_compute_instance.example["Standard_D4s_v3"] - └─ Instance usage (Linux, pay as you go, Standard_D4s_v3) 730 hours $175.20 + └─ Instance usage (Linux, pay as you go, Standard_D4s_v3) 730 hours $140.16 azurerm_machine_learning_compute_instance.example["Standard_F4s_v2"] - └─ Instance usage (Linux, pay as you go, Standard_F4s_v2) 730 hours $141.62 + └─ Instance usage (Linux, pay as you go, Standard_F4s_v2) 730 hours $123.37 azurerm_machine_learning_compute_instance.example["Standard_D2s_v3"] - └─ Instance usage (Linux, pay as you go, Standard_D2s_v3) 730 hours $87.60 + └─ Instance usage (Linux, pay as you go, Standard_D2s_v3) 730 hours $70.08 azurerm_machine_learning_compute_instance.with_monthly_hrs - └─ Instance usage (Linux, pay as you go, Standard_D2s_v3) 100 hours $12.00 + └─ Instance usage (Linux, pay as you go, Standard_D2s_v3) 100 hours $9.60 azurerm_storage_account.example ├─ Capacity Monthly cost depends on usage: $0.0392 per GB @@ -63,7 +63,7 @@ ├─ All other operations Monthly cost depends on usage: $0.0043 per 10k operations └─ Blob index Monthly cost depends on usage: $0.075 per 10k tags - OVERALL TOTAL $17,164.81 + OVERALL TOTAL $13,908.07 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -75,5 +75,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $17,165 ┃ - ┃ $17,165 ┃ +┃ main ┃ $13,908 ┃ - ┃ $13,908 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/machine_learning_compute_instance_test/machine_learning_compute_instance_test.tf b/internal/providers/terraform/azure/testdata/machine_learning_compute_instance_test/machine_learning_compute_instance_test.tf index 309fb611fda..e1967cc27ed 100644 --- a/internal/providers/terraform/azure/testdata/machine_learning_compute_instance_test/machine_learning_compute_instance_test.tf +++ b/internal/providers/terraform/azure/testdata/machine_learning_compute_instance_test/machine_learning_compute_instance_test.tf @@ -56,7 +56,6 @@ resource "azurerm_machine_learning_compute_instance" "example" { for_each = toset(local.vm_sizes) name = "example-instance" - location = azurerm_resource_group.example.location machine_learning_workspace_id = azurerm_machine_learning_workspace.example.id virtual_machine_size = each.value @@ -64,7 +63,6 @@ resource "azurerm_machine_learning_compute_instance" "example" { resource "azurerm_machine_learning_compute_instance" "with_monthly_hrs" { name = "with-monthly-hrs" - location = azurerm_resource_group.example.location machine_learning_workspace_id = azurerm_machine_learning_workspace.example.id virtual_machine_size = "Standard_D2s_v3" } diff --git a/internal/providers/terraform/azure/testdata/monitor_data_collection_rule_test/monitor_data_collection_rule_test.tf b/internal/providers/terraform/azure/testdata/monitor_data_collection_rule_test/monitor_data_collection_rule_test.tf index 993fe1cd75c..e942557b010 100644 --- a/internal/providers/terraform/azure/testdata/monitor_data_collection_rule_test/monitor_data_collection_rule_test.tf +++ b/internal/providers/terraform/azure/testdata/monitor_data_collection_rule_test/monitor_data_collection_rule_test.tf @@ -38,6 +38,7 @@ resource "azurerm_monitor_data_collection_rule" "example" { data_sources { syslog { + streams = ["Microsoft-Syslog"] facility_names = ["*"] log_levels = ["*"] name = "test-datasource-syslog" diff --git a/internal/providers/terraform/azure/testdata/mssql_database_test/mssql_database_test.tf b/internal/providers/terraform/azure/testdata/mssql_database_test/mssql_database_test.tf index d2bd9e5376d..1eafe6b8708 100644 --- a/internal/providers/terraform/azure/testdata/mssql_database_test/mssql_database_test.tf +++ b/internal/providers/terraform/azure/testdata/mssql_database_test/mssql_database_test.tf @@ -8,7 +8,7 @@ resource "azurerm_resource_group" "example" { location = "eastus" } -resource "azurerm_sql_server" "example" { +resource "azurerm_mssql_server" "example" { name = "example-sqlserver" resource_group_name = azurerm_resource_group.example.name location = "eastus" @@ -19,116 +19,116 @@ resource "azurerm_sql_server" "example" { resource "azurerm_mssql_database" "general_purpose_gen_without_license" { name = "acctest-db-d" - server_id = azurerm_sql_server.example.id + server_id = azurerm_mssql_server.example.id sku_name = "GP_Gen5_4" license_type = "BasePrice" } resource "azurerm_mssql_database" "business_critical_gen" { name = "acctest-db-d" - server_id = azurerm_sql_server.example.id + server_id = azurerm_mssql_server.example.id sku_name = "BC_Gen5_8" max_size_gb = 10 } resource "azurerm_mssql_database" "business_critical_m" { name = "acctest-db-d" - server_id = azurerm_sql_server.example.id + server_id = azurerm_mssql_server.example.id sku_name = "BC_M_8" max_size_gb = 50 } resource "azurerm_mssql_database" "hyperscale_gen" { name = "acctest-db-d" - server_id = azurerm_sql_server.example.id + server_id = azurerm_mssql_server.example.id sku_name = "HS_Gen5_2" max_size_gb = 100 } resource "azurerm_mssql_database" "hyperscale_gen_with_replicas" { name = "acctest-db-d" - server_id = azurerm_sql_server.example.id + server_id = azurerm_mssql_server.example.id sku_name = "HS_Gen5_2" read_replica_count = 2 } resource "azurerm_mssql_database" "general_purpose_gen" { name = "acctest-db-d" - server_id = azurerm_sql_server.example.id + server_id = azurerm_mssql_server.example.id sku_name = "GP_Gen5_4" license_type = "LicenseIncluded" } resource "azurerm_mssql_database" "general_purpose_gen_zone" { name = "acctest-db-d" - server_id = azurerm_sql_server.example.id + server_id = azurerm_mssql_server.example.id sku_name = "GP_Gen5_4" zone_redundant = true } resource "azurerm_mssql_database" "serverless" { name = "acctest-db-d" - server_id = azurerm_sql_server.example.id + server_id = azurerm_mssql_server.example.id sku_name = "GP_S_Gen5_4" } resource "azurerm_mssql_database" "serverless_zone" { name = "acctest-db-d" - server_id = azurerm_sql_server.example.id + server_id = azurerm_mssql_server.example.id sku_name = "GP_S_Gen5_4" zone_redundant = true } resource "azurerm_mssql_database" "backup_default" { name = "acctest-db-d" - server_id = azurerm_sql_server.example.id + server_id = azurerm_mssql_server.example.id sku_name = "GP_Gen5_4" } resource "azurerm_mssql_database" "backup_geo" { name = "acctest-db-d" - server_id = azurerm_sql_server.example.id + server_id = azurerm_mssql_server.example.id sku_name = "GP_Gen5_4" storage_account_type = "Geo" } resource "azurerm_mssql_database" "backup_zone" { name = "acctest-db-d" - server_id = azurerm_sql_server.example.id + server_id = azurerm_mssql_server.example.id sku_name = "GP_Gen5_4" storage_account_type = "Zone" } resource "azurerm_mssql_database" "backup_local" { name = "acctest-db-d" - server_id = azurerm_sql_server.example.id + server_id = azurerm_mssql_server.example.id sku_name = "GP_Gen5_4" storage_account_type = "Local" } resource "azurerm_mssql_database" "standard1" { name = "acctest-db-d" - server_id = azurerm_sql_server.example.id + server_id = azurerm_mssql_server.example.id sku_name = "S1" } resource "azurerm_mssql_database" "standard12" { name = "acctest-db-d" - server_id = azurerm_sql_server.example.id + server_id = azurerm_mssql_server.example.id sku_name = "S12" max_size_gb = 500 } resource "azurerm_mssql_database" "premium6" { name = "acctest-db-d" - server_id = azurerm_sql_server.example.id + server_id = azurerm_mssql_server.example.id sku_name = "P6" } resource "azurerm_mssql_database" "blank_sku" { name = "acctest-db-e" - server_id = azurerm_sql_server.example.id + server_id = azurerm_mssql_server.example.id } resource "azurerm_mssql_database" "elastic_pool" { name = "acctest-db-f" - server_id = azurerm_sql_server.example.id + server_id = azurerm_mssql_server.example.id sku_name = "ElasticPool" } diff --git a/internal/providers/terraform/azure/testdata/mssql_elasticpool_test/mssql_elasticpool_test.tf b/internal/providers/terraform/azure/testdata/mssql_elasticpool_test/mssql_elasticpool_test.tf index 8d8b5d0e792..c22189c428c 100644 --- a/internal/providers/terraform/azure/testdata/mssql_elasticpool_test/mssql_elasticpool_test.tf +++ b/internal/providers/terraform/azure/testdata/mssql_elasticpool_test/mssql_elasticpool_test.tf @@ -8,7 +8,7 @@ resource "azurerm_resource_group" "example" { location = "West Europe" } -resource "azurerm_sql_server" "example" { +resource "azurerm_mssql_server" "example" { name = "myexamplesqlserver" resource_group_name = azurerm_resource_group.example.name location = "eastus" @@ -21,7 +21,7 @@ resource "azurerm_mssql_elasticpool" "gp_gen5" { name = "gp-gen5" resource_group_name = azurerm_resource_group.example.name location = azurerm_resource_group.example.location - server_name = azurerm_sql_server.example.name + server_name = azurerm_mssql_server.example.name license_type = "LicenseIncluded" max_size_gb = 100 @@ -42,7 +42,7 @@ resource "azurerm_mssql_elasticpool" "gp_gen5_zone_redundant" { name = "gp-gen5" resource_group_name = azurerm_resource_group.example.name location = azurerm_resource_group.example.location - server_name = azurerm_sql_server.example.name + server_name = azurerm_mssql_server.example.name license_type = "LicenseIncluded" zone_redundant = true max_size_gb = 100 @@ -64,7 +64,7 @@ resource "azurerm_mssql_elasticpool" "gp_gen5_zone_no_license" { name = "gp-gen5" resource_group_name = azurerm_resource_group.example.name location = azurerm_resource_group.example.location - server_name = azurerm_sql_server.example.name + server_name = azurerm_mssql_server.example.name license_type = "BasePrice" max_size_gb = 100 @@ -85,7 +85,7 @@ resource "azurerm_mssql_elasticpool" "bc_dc" { name = "bc-dc" resource_group_name = azurerm_resource_group.example.name location = azurerm_resource_group.example.location - server_name = azurerm_sql_server.example.name + server_name = azurerm_mssql_server.example.name license_type = "LicenseIncluded" max_size_gb = 100 @@ -106,7 +106,7 @@ resource "azurerm_mssql_elasticpool" "bc_dc_zone_redundant" { name = "bc-dc-zone-redundant" resource_group_name = azurerm_resource_group.example.name location = azurerm_resource_group.example.location - server_name = azurerm_sql_server.example.name + server_name = azurerm_mssql_server.example.name license_type = "LicenseIncluded" max_size_gb = 100 zone_redundant = true @@ -128,7 +128,7 @@ resource "azurerm_mssql_elasticpool" "basic_100" { name = "basic-100" resource_group_name = azurerm_resource_group.example.name location = azurerm_resource_group.example.location - server_name = azurerm_sql_server.example.name + server_name = azurerm_mssql_server.example.name max_size_gb = 9.7656250 sku { @@ -147,7 +147,7 @@ resource "azurerm_mssql_elasticpool" "standard_200" { name = "standard-200" resource_group_name = azurerm_resource_group.example.name location = azurerm_resource_group.example.location - server_name = azurerm_sql_server.example.name + server_name = azurerm_mssql_server.example.name max_size_gb = 300 # 100 GB extra storage sku { @@ -166,7 +166,7 @@ resource "azurerm_mssql_elasticpool" "premium_500" { name = "premium-500" resource_group_name = azurerm_resource_group.example.name location = azurerm_resource_group.example.location - server_name = azurerm_sql_server.example.name + server_name = azurerm_mssql_server.example.name max_size_gb = 1024 # 274 GB extra storage sku { diff --git a/internal/providers/terraform/azure/testdata/public_ip_test/public_ip_test.golden b/internal/providers/terraform/azure/testdata/public_ip_test/public_ip_test.golden index c1aa97c3743..8c1fc396f32 100644 --- a/internal/providers/terraform/azure/testdata/public_ip_test/public_ip_test.golden +++ b/internal/providers/terraform/azure/testdata/public_ip_test/public_ip_test.golden @@ -4,13 +4,13 @@ azurerm_public_ip.example └─ IP address (static, regional) 730 hours $3.65 - azurerm_public_ip.global - └─ IP address (static, global) 730 hours $3.65 - azurerm_public_ip.example1 └─ IP address (dynamic, regional) 730 hours $2.92 - OVERALL TOTAL $10.22 + azurerm_public_ip.global + └─ IP address (static, global) 730 hours not found + + OVERALL TOTAL $6.57 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -22,5 +22,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $10 ┃ - ┃ $10 ┃ +┃ main ┃ $7 ┃ - ┃ $7 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/service_plan_test/service_plan_test.golden b/internal/providers/terraform/azure/testdata/service_plan_test/service_plan_test.golden index 4b4652ac126..2e905d86492 100644 --- a/internal/providers/terraform/azure/testdata/service_plan_test/service_plan_test.golden +++ b/internal/providers/terraform/azure/testdata/service_plan_test/service_plan_test.golden @@ -466,9 +466,6 @@ azurerm_service_plan.example["Windows.I2.1"] └─ Instance usage (I2) 730 hours $438.00 - azurerm_service_plan.example["Windows.P0v3.3"] - └─ Instance usage (P0v3) 2,190 hours $438.00 - azurerm_service_plan.example["Windows.P1v2.3"] └─ Instance usage (P1v2) 2,190 hours $438.00 @@ -484,9 +481,6 @@ azurerm_service_plan.example["WindowsContainer.I2.1"] └─ Instance usage (I2) 730 hours $438.00 - azurerm_service_plan.example["WindowsContainer.P0v3.3"] - └─ Instance usage (P0v3) 2,190 hours $438.00 - azurerm_service_plan.example["WindowsContainer.P1v2.3"] └─ Instance usage (P1v2) 2,190 hours $438.00 @@ -505,6 +499,12 @@ azurerm_service_plan.example["WindowsContainer.I1v2.1"] └─ Instance usage (I1v2) 730 hours $399.31 + azurerm_service_plan.example["Windows.P0v3.3"] + └─ Instance usage (P0v3) 2,190 hours $344.93 + + azurerm_service_plan.example["WindowsContainer.P0v3.3"] + └─ Instance usage (P0v3) 2,190 hours $344.93 + azurerm_service_plan.example["Linux.P1v3.3"] └─ Instance usage (P1v3) 2,190 hours $339.45 @@ -520,9 +520,6 @@ azurerm_service_plan.example["Linux.P3v2.1"] └─ Instance usage (P3v2) 730 hours $294.19 - azurerm_service_plan.example["Windows.P0v3.2"] - └─ Instance usage (P0v3) 1,460 hours $292.00 - azurerm_service_plan.example["Windows.P1v2.2"] └─ Instance usage (P1v2) 1,460 hours $292.00 @@ -535,9 +532,6 @@ azurerm_service_plan.example["Windows.S3.1"] └─ Instance usage (S3) 730 hours $292.00 - azurerm_service_plan.example["WindowsContainer.P0v3.2"] - └─ Instance usage (P0v3) 1,460 hours $292.00 - azurerm_service_plan.example["WindowsContainer.P1v2.2"] └─ Instance usage (P1v2) 1,460 hours $292.00 @@ -571,9 +565,15 @@ azurerm_service_plan.example["WindowsContainer.P1mv3.1"] └─ Instance usage (P1mv3) 730 hours $252.58 + azurerm_service_plan.example["Windows.P0v3.2"] + └─ Instance usage (P0v3) 1,460 hours $229.95 + azurerm_service_plan.example["Windows.P1v3.1"] └─ Instance usage (P1v3) 730 hours $229.95 + azurerm_service_plan.example["WindowsContainer.P0v3.2"] + └─ Instance usage (P0v3) 1,460 hours $229.95 + azurerm_service_plan.example["WindowsContainer.P1v3.1"] └─ Instance usage (P1v3) 730 hours $229.95 @@ -583,9 +583,6 @@ azurerm_service_plan.example["Linux.P2v3.1"] └─ Instance usage (P2v3) 730 hours $226.30 - azurerm_service_plan.example["Linux.P0v3.3"] - └─ Instance usage (P0v3) 2,190 hours $221.19 - azurerm_service_plan.example["Linux.P1v2.3"] └─ Instance usage (P1v2) 2,190 hours $221.19 @@ -619,15 +616,15 @@ azurerm_service_plan.example["Linux.S1.3"] └─ Instance usage (S1) 2,190 hours $208.05 + azurerm_service_plan.example["Linux.P0v3.3"] + └─ Instance usage (P0v3) 2,190 hours $169.73 + azurerm_service_plan.example["Windows.B1.3"] └─ Instance usage (B1) 2,190 hours $164.25 azurerm_service_plan.example["WindowsContainer.B1.3"] └─ Instance usage (B1) 2,190 hours $164.25 - azurerm_service_plan.example["Linux.P0v3.2"] - └─ Instance usage (P0v3) 1,460 hours $147.46 - azurerm_service_plan.example["Linux.P1v2.2"] └─ Instance usage (P1v2) 1,460 hours $147.46 @@ -637,9 +634,6 @@ azurerm_service_plan.example["Linux.B3.3"] └─ Instance usage (B3) 2,190 hours $146.73 - azurerm_service_plan.example["Windows.P0v3.1"] - └─ Instance usage (P0v3) 730 hours $146.00 - azurerm_service_plan.example["Windows.P1v2.1"] └─ Instance usage (P1v2) 730 hours $146.00 @@ -649,9 +643,6 @@ azurerm_service_plan.example["Windows.S2.1"] └─ Instance usage (S2) 730 hours $146.00 - azurerm_service_plan.example["WindowsContainer.P0v3.1"] - └─ Instance usage (P0v3) 730 hours $146.00 - azurerm_service_plan.example["WindowsContainer.P1v2.1"] └─ Instance usage (P1v2) 730 hours $146.00 @@ -670,6 +661,15 @@ azurerm_service_plan.example["Linux.P1mv3.1"] └─ Instance usage (P1mv3) 730 hours $135.78 + azurerm_service_plan.example["Windows.P0v3.1"] + └─ Instance usage (P0v3) 730 hours $114.98 + + azurerm_service_plan.example["WindowsContainer.P0v3.1"] + └─ Instance usage (P0v3) 730 hours $114.98 + + azurerm_service_plan.example["Linux.P0v3.2"] + └─ Instance usage (P0v3) 1,460 hours $113.15 + azurerm_service_plan.example["Linux.P1v3.1"] └─ Instance usage (P1v3) 730 hours $113.15 @@ -691,9 +691,6 @@ azurerm_service_plan.example["Linux.B2.3"] └─ Instance usage (B2) 2,190 hours $74.46 - azurerm_service_plan.example["Linux.P0v3.1"] - └─ Instance usage (P0v3) 730 hours $73.73 - azurerm_service_plan.example["Linux.P1v2.1"] └─ Instance usage (P1v2) 730 hours $73.73 @@ -706,6 +703,9 @@ azurerm_service_plan.example["Linux.S1.1"] └─ Instance usage (S1) 730 hours $69.35 + azurerm_service_plan.example["Linux.P0v3.1"] + └─ Instance usage (P0v3) 730 hours $56.58 + azurerm_service_plan.example["Windows.B1.1"] └─ Instance usage (B1) 730 hours $54.75 @@ -811,7 +811,7 @@ azurerm_service_plan.example["WindowsContainer.F1.3"] └─ Instance usage (F1) 2,190 hours $0.00 - OVERALL TOTAL $611,320.98 + OVERALL TOTAL $610,845.75 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -823,5 +823,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $611,321 ┃ - ┃ $611,321 ┃ +┃ main ┃ $610,846 ┃ - ┃ $610,846 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/synapse_spark_pool_test/synapse_spark_pool_test.tf b/internal/providers/terraform/azure/testdata/synapse_spark_pool_test/synapse_spark_pool_test.tf index efa48450897..c48b94de6c2 100644 --- a/internal/providers/terraform/azure/testdata/synapse_spark_pool_test/synapse_spark_pool_test.tf +++ b/internal/providers/terraform/azure/testdata/synapse_spark_pool_test/synapse_spark_pool_test.tf @@ -33,12 +33,6 @@ resource "azurerm_synapse_workspace" "example" { sql_administrator_login_password = "H@Sh1CoR3!" managed_virtual_network_enabled = false - aad_admin { - login = "AzureAD Admin" - object_id = "00000000-0000-0000-0000-000000000000" - tenant_id = "00000000-0000-0000-0000-000000000000" - } - identity { type = "SystemAssigned" } @@ -53,6 +47,7 @@ resource "azurerm_synapse_spark_pool" "default" { synapse_workspace_id = azurerm_synapse_workspace.example.id node_size_family = "MemoryOptimized" node_size = "Small" + spark_version = "3.4" node_count = 3 auto_pause { @@ -65,6 +60,7 @@ resource "azurerm_synapse_spark_pool" "autoscale" { synapse_workspace_id = azurerm_synapse_workspace.example.id node_size_family = "MemoryOptimized" node_size = "Small" + spark_version = "3.4" auto_scale { min_node_count = 4 diff --git a/internal/providers/terraform/azure/testdata/synapse_sql_pool_test/synapse_sql_pool_test.tf b/internal/providers/terraform/azure/testdata/synapse_sql_pool_test/synapse_sql_pool_test.tf index d98276ec292..bc3781355bf 100644 --- a/internal/providers/terraform/azure/testdata/synapse_sql_pool_test/synapse_sql_pool_test.tf +++ b/internal/providers/terraform/azure/testdata/synapse_sql_pool_test/synapse_sql_pool_test.tf @@ -33,12 +33,6 @@ resource "azurerm_synapse_workspace" "example" { sql_administrator_login_password = "H@Sh1CoR3!" managed_virtual_network_enabled = false - aad_admin { - login = "AzureAD Admin" - object_id = "00000000-0000-0000-0000-000000000000" - tenant_id = "00000000-0000-0000-0000-000000000000" - } - identity { type = "SystemAssigned" } @@ -53,6 +47,7 @@ resource "azurerm_synapse_sql_pool" "default" { synapse_workspace_id = azurerm_synapse_workspace.example.id sku_name = "DW200c" create_mode = "Default" + storage_account_type = "GRS" } resource "azurerm_synapse_sql_pool" "storage" { @@ -60,6 +55,7 @@ resource "azurerm_synapse_sql_pool" "storage" { synapse_workspace_id = azurerm_synapse_workspace.example.id sku_name = "DW200c" create_mode = "Default" + storage_account_type = "GRS" } resource "azurerm_synapse_sql_pool" "no_backup" { @@ -67,4 +63,5 @@ resource "azurerm_synapse_sql_pool" "no_backup" { synapse_workspace_id = azurerm_synapse_workspace.example.id sku_name = "DW200c" create_mode = "Default" + storage_account_type = "GRS" } diff --git a/internal/providers/terraform/azure/testdata/synapse_workspace_test/synapse_workspace_test.tf b/internal/providers/terraform/azure/testdata/synapse_workspace_test/synapse_workspace_test.tf index 4bee4e6e656..320e55d45a1 100644 --- a/internal/providers/terraform/azure/testdata/synapse_workspace_test/synapse_workspace_test.tf +++ b/internal/providers/terraform/azure/testdata/synapse_workspace_test/synapse_workspace_test.tf @@ -33,12 +33,6 @@ resource "azurerm_synapse_workspace" "general" { sql_administrator_login_password = "H@Sh1CoR3!" managed_virtual_network_enabled = false - aad_admin { - login = "AzureAD Admin" - object_id = "00000000-0000-0000-0000-000000000000" - tenant_id = "00000000-0000-0000-0000-000000000000" - } - identity { type = "SystemAssigned" } @@ -57,12 +51,6 @@ resource "azurerm_synapse_workspace" "vnet" { sql_administrator_login_password = "H@Sh1CoR3!" managed_virtual_network_enabled = true - aad_admin { - login = "AzureAD Admin" - object_id = "00000000-0000-0000-0000-000000000000" - tenant_id = "00000000-0000-0000-0000-000000000000" - } - identity { type = "SystemAssigned" } @@ -80,13 +68,6 @@ resource "azurerm_synapse_workspace" "datapipelines" { sql_administrator_login = "sqladminuser" sql_administrator_login_password = "H@Sh1CoR3!" - - aad_admin { - login = "AzureAD Admin" - object_id = "00000000-0000-0000-0000-000000000000" - tenant_id = "00000000-0000-0000-0000-000000000000" - } - identity { type = "SystemAssigned" } @@ -104,13 +85,6 @@ resource "azurerm_synapse_workspace" "dataflows" { sql_administrator_login = "sqladminuser" sql_administrator_login_password = "H@Sh1CoR3!" - - aad_admin { - login = "AzureAD Admin" - object_id = "00000000-0000-0000-0000-000000000000" - tenant_id = "00000000-0000-0000-0000-000000000000" - } - identity { type = "SystemAssigned" } diff --git a/internal/providers/terraform/azure/testdata/virtual_machine_test/virtual_machine_test.golden b/internal/providers/terraform/azure/testdata/virtual_machine_test/virtual_machine_test.golden index d62ed298c49..952ab6e557b 100644 --- a/internal/providers/terraform/azure/testdata/virtual_machine_test/virtual_machine_test.golden +++ b/internal/providers/terraform/azure/testdata/virtual_machine_test/virtual_machine_test.golden @@ -925,22 +925,22 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_D5_v2"] - ├─ Instance usage (Windows, pay as you go, Standard_D5_v2) 730 hours $1,471.68 + azurerm_virtual_machine.windows_vms["Standard_DS5_v2"] + ├─ Instance usage (Windows, pay as you go, Standard_DS5_v2) 730 hours $1,471.68 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_DS5_v2"] - ├─ Instance usage (Windows, pay as you go, Standard_DS5_v2) 730 hours $1,471.68 + azurerm_virtual_machine.linux_vms["Standard_A9"] + ├─ Instance usage (Linux, pay as you go, Standard_A9) 730 hours $1,423.50 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.linux_vms["Standard_A9"] - ├─ Instance usage (Linux, pay as you go, Standard_A9) 730 hours $1,423.50 + azurerm_virtual_machine.windows_vms["Standard_D5_v2"] + ├─ Instance usage (Windows, pay as you go, Standard_D5_v2) 730 hours $1,389.92 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 @@ -1693,7 +1693,7 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - OVERALL TOTAL $1,039,546.32 + OVERALL TOTAL $1,039,464.56 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -1705,5 +1705,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $1,039,546 ┃ $0.55 ┃ $1,039,546 ┃ +┃ main ┃ $1,039,464 ┃ $0.55 ┃ $1,039,465 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/virtual_network_gateway_connection_test/virtual_network_gateway_connection_test.golden b/internal/providers/terraform/azure/testdata/virtual_network_gateway_connection_test/virtual_network_gateway_connection_test.golden index 53ee88be933..01cd28254bd 100644 --- a/internal/providers/terraform/azure/testdata/virtual_network_gateway_connection_test/virtual_network_gateway_connection_test.golden +++ b/internal/providers/terraform/azure/testdata/virtual_network_gateway_connection_test/virtual_network_gateway_connection_test.golden @@ -55,9 +55,9 @@ └─ VPN gateway (VpnGw5) 730 hours $10.95 azurerm_public_ip.example - └─ IP address (dynamic, regional) 730 hours $2.92 + └─ IP address (dynamic, regional) 730 hours not found - OVERALL TOTAL $5,964.83 + OVERALL TOTAL $5,961.91 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -69,5 +69,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $5,965 ┃ - ┃ $5,965 ┃ +┃ main ┃ $5,962 ┃ - ┃ $5,962 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/virtual_network_gateway_test/virtual_network_gateway_test.golden b/internal/providers/terraform/azure/testdata/virtual_network_gateway_test/virtual_network_gateway_test.golden index b37532eb3f4..f1f1924fa69 100644 --- a/internal/providers/terraform/azure/testdata/virtual_network_gateway_test/virtual_network_gateway_test.golden +++ b/internal/providers/terraform/azure/testdata/virtual_network_gateway_test/virtual_network_gateway_test.golden @@ -37,9 +37,9 @@ └─ VPN gateway data tranfer Monthly cost depends on usage: $0.035 per GB azurerm_public_ip.example - └─ IP address (dynamic, regional) 730 hours $2.92 + └─ IP address (dynamic, regional) 730 hours not found - OVERALL TOTAL $49,247.27 + OVERALL TOTAL $49,244.35 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -51,5 +51,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $49,247 ┃ - ┃ $49,247 ┃ +┃ main ┃ $49,244 ┃ - ┃ $49,244 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/google/testdata/cloudfunctions_function_test/cloudfunctions_function_test.golden b/internal/providers/terraform/google/testdata/cloudfunctions_function_test/cloudfunctions_function_test.golden index ff155e2e473..a4e134b0b16 100644 --- a/internal/providers/terraform/google/testdata/cloudfunctions_function_test/cloudfunctions_function_test.golden +++ b/internal/providers/terraform/google/testdata/cloudfunctions_function_test/cloudfunctions_function_test.golden @@ -1,19 +1,19 @@ - Name Monthly Qty Unit Monthly Cost - - google_cloudfunctions_function.my_function - ├─ CPU 1,200,000 GHz-seconds $12.00 * - ├─ Memory 750,000 GB-seconds $1.88 * - ├─ Invocations 10,000,000 invocations $4.00 * - └─ Outbound data transfer 100 GB $12.00 * - - google_cloudfunctions_function.function - ├─ CPU Monthly cost depends on usage: $0.00001 per GHz-seconds - ├─ Memory Monthly cost depends on usage: $0.0000025 per GB-seconds - ├─ Invocations Monthly cost depends on usage: $0.0000004 per invocations - └─ Outbound data transfer Monthly cost depends on usage: $0.12 per GB - - OVERALL TOTAL $29.88 + Name Monthly Qty Unit Monthly Cost + + google_cloudfunctions_function.my_function + ├─ CPU 1,200,000 GHz-seconds not found * + ├─ Memory 750,000 GB-seconds not found * + ├─ Invocations 10,000,000 invocations not found * + └─ Outbound data transfer 100 GB not found * + + google_cloudfunctions_function.function + ├─ CPU not found + ├─ Memory not found + ├─ Invocations not found + └─ Outbound data transfer not found + + OVERALL TOTAL $0.00 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -24,5 +24,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $0.00 ┃ $30 ┃ $30 ┃ +┃ main ┃ $0.00 ┃ - ┃ $0.00 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file From e7897f2543026dedefda0fc3a32a01ded0a8b7cf Mon Sep 17 00:00:00 2001 From: Hugo Rut Date: Fri, 30 Aug 2024 13:34:16 +0200 Subject: [PATCH 02/81] fix: speed up parent Terragrunt config detection (#3186) This PR changes the parent Terragrunt config detection to parse the list of detected config files into a Tree before trying to determine which Terragrunt files are parent configuration. This significantly speeds up the time to detect parent configs as we need many less io operations to determine which files are parents. On a large repo with ~6k Terragrunt config files the speed of infracost generate went from ~50s to ~1.5s after this change. Co-authored-by: Owen --- internal/hcl/project_locator.go | 169 +++++++++++++++++++++----------- 1 file changed, 113 insertions(+), 56 deletions(-) diff --git a/internal/hcl/project_locator.go b/internal/hcl/project_locator.go index 73a801fb119..1ee8d416e3f 100644 --- a/internal/hcl/project_locator.go +++ b/internal/hcl/project_locator.go @@ -584,6 +584,19 @@ func (t *TreeNode) ParentNode() *TreeNode { return nil } +// Parents returns the list of parent nodes of the current node. +func (t *TreeNode) Parents() []*TreeNode { + var parents []*TreeNode + parent := t.ParentNode() + + for parent != nil { + parents = append(parents, parent) + parent = parent.ParentNode() + } + + return parents +} + // UnusedParentVarFiles returns a list of any parent directories that contain var // files that have not been used by a project. func (t *TreeNode) UnusedParentVarFiles() []*VarFiles { @@ -962,9 +975,10 @@ type RootPath struct { // TerraformVarFiles are a list of any .tfvars or .tfvars.json files found at the root level. TerraformVarFiles RootPathVarFiles - HasChildVarFiles bool - IsTerragrunt bool - ModuleCalls []string + HasChildVarFiles bool + IsTerragrunt bool + IsParentTerragruntConfig bool + ModuleCalls []string } func (r *RootPath) RelPath() string { @@ -1146,6 +1160,7 @@ func (p *ProjectLocator) FindRootModules(startingPath string) []RootPath { p.findTerragruntDirs(startingPath) p.walkPaths(startingPath, 0, p.maxSearchDepth()) + for _, project := range p.discoveredProjects { if _, ok := p.projectDuplicates[project.path]; ok { p.projectDuplicates[project.path] = true @@ -1661,6 +1676,7 @@ func (p *ProjectLocator) findTerragruntDirs(fullPath string) { terragruntConfigFiles, err := tgconfig.FindConfigFilesInPath(fullPath, &options.TerragruntOptions{ DownloadDir: terragruntDownloadDir, }) + if err != nil { p.logger.Debug().Err(err).Msgf("failed to find terragrunt files in path %s", fullPath) } @@ -1669,16 +1685,107 @@ func (p *ProjectLocator) findTerragruntDirs(fullPath string) { p.wdContainsTerragrunt = true } - for _, configFile := range terragruntConfigFiles { - if !p.shouldSkipDir(filepath.Dir(configFile)) && !IsParentTerragruntConfig(configFile, terragruntConfigFiles) { + for _, project := range p.removeParentTerragruntFiles(fullPath, terragruntConfigFiles) { + if !p.shouldSkipDir(filepath.Dir(project)) { p.discoveredProjects = append(p.discoveredProjects, discoveredProject{ - path: filepath.Dir(configFile), + path: project, isTerragrunt: true, }) } } } +// removeParentTerragruntFiles removes any parent Terragrunt config files from +// the list of discovered Terragrunt configuration files. +func (p *ProjectLocator) removeParentTerragruntFiles(startingPath string, files []string) []string { + var paths []RootPath + nameMap := make(map[string]string) + for _, file := range files { + dir := filepath.Dir(file) + nameMap[dir] = filepath.Base(file) + + paths = append(paths, RootPath{ + DetectedPath: dir, + StartingPath: startingPath, + }) + } + + var projects []string + root := CreateTreeNode(startingPath, paths, nil, nil) + + // We need to slightly modify the TreeNode behaviour so that it works for this Terragrunt + // use case. For this case if there is a detected Terragrunt config file in the root of the + // directory we need to set it on the Root node so that we can traverse the tree and mark + // all parent nodes as parent Terragrunt config files. + var rootChildren []*TreeNode + for _, child := range root.Children { + if child.Name == "." && child.RootPath != nil { + root.RootPath = child.RootPath + } else { + rootChildren = append(rootChildren, child) + } + } + root.Children = rootChildren + + root.PostOrder(func(t *TreeNode) { + if p.shouldSkipDir(filepath.Dir(t.RootPath.DetectedPath)) { + return + } + + if t.RootPath.IsParentTerragruntConfig { + // If the current node is a parent terragrunt config, we can skip processing it. + return + } + + parent := t.ParentNode() + if parent == nil { + // no valid parent node so no need to process this child node. + return + } + + if parent.RootPath != nil && parent.RootPath.IsParentTerragruntConfig { + // no need to process this child node as the parent has already been marked as a parent terragrunt config. + return + } + + filename := nameMap[t.RootPath.DetectedPath] + if p.processFile(filepath.Join(t.RootPath.DetectedPath, filename)) { + // Mark all parent nodes as parent terragrunt config of the current node. + parents := t.Parents() + for _, v := range parents { + if v != nil && v.RootPath != nil { + v.RootPath.IsParentTerragruntConfig = true + } + } + } + }) + + root.Visit(func(t *TreeNode) { + if t.RootPath != nil && !t.RootPath.IsParentTerragruntConfig { + projects = append(projects, t.RootPath.DetectedPath) + } + }) + + return projects +} + +func (p *ProjectLocator) processFile(name string) bool { + file, err := os.Open(name) + if err != nil { + return false + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + if strings.Contains(scanner.Text(), "find_in_parent_folders()") { + return true + } + } + + return false +} + func buildDirMatcher(dirs []string, fullPath string) func(string) bool { var rawMatches []string globMatches := make(map[string]struct{}) @@ -1718,53 +1825,3 @@ func buildDirMatcher(dirs []string, fullPath string) func(string) bool { return false } } - -// IsParentTerragruntConfig checks if a terragrunt config entry is a parent file that is referenced by another config -// with a find_in_parent_folders call. The find_in_parent_folders function searches up the directory tree -// from the file and returns the absolute path to the first terragrunt.hcl. This means if it is found -// we can treat this file as a child terragrunt.hcl. -func IsParentTerragruntConfig(parent string, configFiles []string) bool { - for _, name := range configFiles { - if !isChildDirectory(parent, name) { - continue - } - - file, err := os.Open(name) - if err != nil { - continue - } - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - // skip any commented out lines - if strings.HasPrefix(line, "#") { - continue - } - - if strings.Contains(line, "find_in_parent_folders()") { - file.Close() - return true - } - } - - file.Close() - } - - return false -} - -func isChildDirectory(parent, child string) bool { - if parent == child { - return false - } - - parentDir := filepath.Dir(parent) - childDir := filepath.Dir(child) - p, err := filepath.Rel(parentDir, childDir) - if err != nil || strings.Contains(p, "..") { - return false - } - - return true -} From 3f15a8e816f454a473ba1f62d14ef988b9b8a6ed Mon Sep 17 00:00:00 2001 From: Salwa <51963397+jumana-s@users.noreply.github.com> Date: Wed, 4 Sep 2024 11:41:19 -0400 Subject: [PATCH 03/81] chore: Upgrade hashicorp/vault/api package to 1.14.0 (#1) (#3187) * Update go.mod * Update go.sum --- go.mod | 23 ++++--------- go.sum | 104 ++++++++------------------------------------------------- 2 files changed, 21 insertions(+), 106 deletions(-) diff --git a/go.mod b/go.mod index feee7d0954c..20b57e7095d 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/tidwall/gjson v1.17.0 github.com/zclconf/go-cty v1.14.1 - golang.org/x/crypto v0.22.0 // indirect + golang.org/x/crypto v0.23.0 // indirect golang.org/x/mod v0.14.0 gopkg.in/go-playground/assert.v1 v1.2.1 gopkg.in/yaml.v2 v2.4.0 @@ -79,7 +79,7 @@ require ( github.com/slack-go/slack v0.12.3 github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect - golang.org/x/text v0.14.0 + golang.org/x/text v0.15.0 golang.org/x/tools v0.16.1 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect ) @@ -144,8 +144,6 @@ require ( github.com/agnivade/levenshtein v1.1.1 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/apparentlymart/go-versions v1.0.1 // indirect - github.com/armon/go-metrics v0.3.10 // indirect - github.com/armon/go-radix v1.0.0 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.25 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.28 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5 // indirect @@ -160,10 +158,10 @@ require ( github.com/dlclark/regexp2 v1.8.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-jose/go-jose/v4 v4.0.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/snappy v0.0.4 // indirect github.com/google/go-github/v35 v35.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect github.com/goware/prefixer v0.0.0-20160118172347-395022866408 // indirect @@ -171,20 +169,16 @@ require ( github.com/gruntwork-io/terratest v0.41.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-hclog v1.6.3 // indirect - github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-plugin v1.4.10 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect - github.com/hashicorp/go-secure-stdlib/mlock v0.1.2 // indirect - github.com/hashicorp/go-secure-stdlib/parseutil v0.1.3 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 // indirect github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect github.com/hashicorp/go-sockaddr v1.0.2 // indirect - github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/logutils v1.0.0 // indirect github.com/hashicorp/terraform v0.15.3 // indirect github.com/hashicorp/terraform-registry-address v0.2.1 // indirect - github.com/hashicorp/vault/api v1.5.0 // indirect - github.com/hashicorp/vault/sdk v0.4.1 // indirect + github.com/hashicorp/vault/api v1.14.0 // indirect github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 // indirect github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect @@ -193,11 +187,10 @@ require ( github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/lib/pq v1.10.5 // indirect github.com/mattn/go-zglob v0.0.3 // indirect - github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/panicwrap v1.0.0 // indirect github.com/oklog/run v1.1.0 // indirect github.com/owenrumney/go-sarif v1.1.1 // indirect - github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect @@ -218,13 +211,11 @@ require ( github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect go.mozilla.org/gopgagent v0.0.0-20170926210634-4d7ea76ff71a // indirect go.mozilla.org/sops/v3 v3.7.3 // indirect - go.uber.org/atomic v1.9.0 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/term v0.21.0 // indirect golang.org/x/time v0.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.66.4 // indirect - gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/urfave/cli.v1 v1.20.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect ) @@ -261,7 +252,7 @@ require ( github.com/yashtewari/glob-intersection v0.1.0 // indirect github.com/zclconf/go-cty-yaml v1.0.3 go.opencensus.io v0.24.0 // indirect - golang.org/x/net v0.24.0 // indirect + golang.org/x/net v0.25.0 // indirect google.golang.org/api v0.114.0 google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect diff --git a/go.sum b/go.sum index 5e05227074d..00ec962b3d8 100644 --- a/go.sum +++ b/go.sum @@ -241,7 +241,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ChrisTrenkamp/goxpath v0.0.0-20170922090931-c385f95c6022/go.mod h1:nuWgzSkT5PnyOd+272uUmV0dnAnAn42Mk7PiQC5VzN4= github.com/ChrisTrenkamp/goxpath v0.0.0-20190607011252-c5096ec8773d/go.mod h1:nuWgzSkT5PnyOd+272uUmV0dnAnAn42Mk7PiQC5VzN4= -github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= @@ -277,9 +276,7 @@ github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVb github.com/alecthomas/jsonschema v0.0.0-20211209230136-e2b41affa5c1 h1:6mZ7MG/flSahicBVy4GKlWI+dzoR5rgnm7H8e17TAio= github.com/alecthomas/jsonschema v0.0.0-20211209230136-e2b41affa5c1/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/aliscott/dag v1.3.2-0.20231115114512-4ce18c825f94 h1:GRH/2nycl5JYrn248Hy/I4h9JrsogYHB02pKqs+ERiQ= github.com/aliscott/dag v1.3.2-0.20231115114512-4ce18c825f94/go.mod h1:OCh6ghKmU0hPjtwMqWBoNxPmtRioKd1xSu7Zs4sbIqM= github.com/aliscott/go-pretty/v6 v6.1.1-0.20210226104003-408905a61c8e h1:D+6DwJEaRT97rBY5Vamed9SzXtm5zXUB28kGVv3nhLM= @@ -312,11 +309,7 @@ github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= -github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8Uo= -github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= @@ -391,7 +384,6 @@ github.com/awslabs/goformation/v7 v7.13.1 h1:QlPn8qwNCqYhrb4GW8kLjT4j1J49n5Qh/an github.com/awslabs/goformation/v7 v7.13.1/go.mod h1:FTCFMNesubEX0LAd6kIR+YkDD1U+5UaMbXtgPUgsck0= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= @@ -408,7 +400,6 @@ github.com/bytecodealliance/wasmtime-go v1.0.0 h1:9u9gqaUiaJeN5IoD1L7egD8atOnTGy github.com/bytecodealliance/wasmtime-go v1.0.0/go.mod h1:jjlqQbWUfVSbehpErw3UoWFndBXRRMvfikYH6KsCwOg= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M= github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -423,8 +414,6 @@ github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXH github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= -github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= @@ -450,7 +439,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= @@ -503,7 +491,6 @@ github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go. github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.5.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -520,10 +507,6 @@ github.com/foxcpp/go-mockdns v0.0.0-20210729171921-fb145fc6f897 h1:E52jfcE64UG42 github.com/foxcpp/go-mockdns v0.0.0-20210729171921-fb145fc6f897/go.mod h1:lgRN6+KxQBawyIghpnl5CezHFGS9VLzvtVlwxvzXTQ4= github.com/franela/goblin v0.0.0-20211003143422-0a4f594942bf h1:NrF81UtW8gG2LBGkXFQFqlfNnvMt9WdB46sfdJY4oqc= github.com/franela/goblin v0.0.0-20211003143422-0a4f594942bf/go.mod h1:VzmDKDJVZI3aJmnRI9VjAn9nJ8qPPsN1fqzr9dqInIo= -github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= -github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU= -github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= -github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fzipp/gocyclo v0.3.1/go.mod h1:DJHO6AUmbdqj2ET4Z9iArSuwWgYDRryYt2wASxc7x3E= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -531,7 +514,6 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= -github.com/go-asn1-ber/asn1-ber v1.3.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= @@ -546,11 +528,10 @@ github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lK github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-jose/go-jose/v4 v4.0.1 h1:QVEPDE3OluqXBQZDcnNvQrInro2h0e4eqNbnZSWqS6U= +github.com/go-jose/go-jose/v4 v4.0.1/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-ldap/ldap/v3 v3.1.10/go.mod h1:5Zun81jBTabRaI8lzN7E1JjyEl1g6zI6u9pd8luAK4Q= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -563,7 +544,6 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= -github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= @@ -741,32 +721,21 @@ github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/S github.com/hashicorp/go-cty-funcs v0.0.0-20230405223818-a090f58aa992 h1:fYOrSfO5C9PmFGtmRWSYGqq52SOoE2dXMtAn2Xzh1LQ= github.com/hashicorp/go-cty-funcs v0.0.0-20230405223818-a090f58aa992/go.mod h1:Abjk0jbRkDaNCzsRhOv2iDCofYpX1eVsjozoiK63qLA= github.com/hashicorp/go-getter v1.5.1/go.mod h1:a7z7NPPfNQpJWcn4rSWFtdrSldqLdLPEF3d8nFMsSLM= -github.com/hashicorp/go-getter v1.7.5 h1:dT58k9hQ/vbxNMwoI5+xFYAJuv6152UNvdHokfI5wE4= -github.com/hashicorp/go-getter v1.7.5/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= github.com/hashicorp/go-getter v1.7.6 h1:5jHuM+aH373XNtXl9TNTUH5Qd69Trve11tHIrB+6yj4= github.com/hashicorp/go-getter v1.7.6/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= -github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v0.15.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v0.0.0-20180129170900-7f3cd4390caa/go.mod h1:6ij3Z20p+OhOkCSrA0gImAWoHYQRGbnlcuk6XYTiaRw= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= -github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-kms-wrapping/entropy v0.1.0/go.mod h1:d1g9WGtAunDNpek8jUIEJnBlbgKS1N2Q61QkHiZyR1g= github.com/hashicorp/go-msgpack v0.5.4/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-plugin v1.4.1/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ= -github.com/hashicorp/go-plugin v1.4.3/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ= github.com/hashicorp/go-plugin v1.4.10 h1:xUbmA4jC6Dq163/fWcp8P3JuHilrHHMLNRxzGQJ9hNk= github.com/hashicorp/go-plugin v1.4.10/go.mod h1:6/1TEzT0eQznvI/gV2CM29DLSkAK/e58mUWKVsPaph0= github.com/hashicorp/go-retryablehttp v0.5.2/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= @@ -774,18 +743,11 @@ github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5O github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= -github.com/hashicorp/go-secure-stdlib/base62 v0.1.1/go.mod h1:EdWO6czbmthiwZ3/PUsDV+UD1D5IRU4ActiaWGwt0Yw= -github.com/hashicorp/go-secure-stdlib/mlock v0.1.1/go.mod h1:zq93CJChV6L9QTfGKtfBxKqD7BqqXx5O04A/ns2p5+I= -github.com/hashicorp/go-secure-stdlib/mlock v0.1.2 h1:p4AKXPPS24tO8Wc8i1gLvSKdmkiSY5xuju57czJ/IJQ= -github.com/hashicorp/go-secure-stdlib/mlock v0.1.2/go.mod h1:zq93CJChV6L9QTfGKtfBxKqD7BqqXx5O04A/ns2p5+I= -github.com/hashicorp/go-secure-stdlib/parseutil v0.1.1/go.mod h1:QmrqtbKuxxSWTN3ETMPuB+VtEiBJ/A9XhoYGv8E1uD8= -github.com/hashicorp/go-secure-stdlib/parseutil v0.1.3 h1:geBw3SBrxQq+buvbf4K+Qltv1gjaXJxy8asD4CjGYow= -github.com/hashicorp/go-secure-stdlib/parseutil v0.1.3/go.mod h1:QmrqtbKuxxSWTN3ETMPuB+VtEiBJ/A9XhoYGv8E1uD8= -github.com/hashicorp/go-secure-stdlib/password v0.1.1/go.mod h1:9hH302QllNwu1o2TGYtSk8I8kTAN0ca1EHpwhm5Mmzo= +github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 h1:om4Al8Oy7kCm/B86rLCLah4Dt5Aa0Fr5rYBG60OzwHQ= +github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6/go.mod h1:QmrqtbKuxxSWTN3ETMPuB+VtEiBJ/A9XhoYGv8E1uD8= github.com/hashicorp/go-secure-stdlib/strutil v0.1.1/go.mod h1:gKOamz3EwoIoJq7mlMIRBpVTAUn8qPCrEclOKKWhD3U= github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= -github.com/hashicorp/go-secure-stdlib/tlsutil v0.1.1/go.mod h1:l8slYwnJA26yBz+ErHpp2IRCLr0vuOMGBORIz4rRiAs= github.com/hashicorp/go-slug v0.4.1/go.mod h1:I5tq5Lv0E2xcNXNkmx7BSfzi1PsJ2cNjs3cC3LwyhK8= github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= @@ -795,7 +757,6 @@ github.com/hashicorp/go-terraform-address v0.0.0-20210506203813-2cc4f0f34da8/go. github.com/hashicorp/go-tfe v0.14.0/go.mod h1:B71izbwmCZdhEo/GzHopCXN3P74cYv2tsff1mxY4J6c= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= @@ -806,13 +767,9 @@ github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mO github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= -github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.6 h1:3xi/Cafd1NaoEnS/yDssIiuVeDVywU0QdFGl3aQaQHM= github.com/hashicorp/golang-lru/v2 v2.0.6/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault h1:UiJeEzCWAYdVaJr8Xo4lBkTozlW1+1yxVUnpbS1xVEk= github.com/hashicorp/hcl v1.0.1-vault/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl/v2 v2.0.0/go.mod h1:oVVDG71tEinNGYCxinCYadcmKU9bglqW9pV3txagJ90= @@ -834,10 +791,8 @@ github.com/hashicorp/terraform-registry-address v0.2.1/go.mod h1:BSE9fIFzp0qWsJU github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734/go.mod h1:kNDNcF7sN4DocDLBkQYz73HGKwN1ANB1blq4lIYLYvg= github.com/hashicorp/terraform-svchost v0.1.1 h1:EZZimZ1GxdqFRinZ1tpJwVxxt49xc/S52uzrw4x0jKQ= github.com/hashicorp/terraform-svchost v0.1.1/go.mod h1:mNsjQfZyf/Jhz35v6/0LWcv26+X7JPS+buii2c9/ctc= -github.com/hashicorp/vault/api v1.5.0 h1:Bp6yc2bn7CWkOrVIzFT/Qurzx528bdavF3nz590eu28= -github.com/hashicorp/vault/api v1.5.0/go.mod h1:LkMdrZnWNrFaQyYYazWVn7KshilfDidgVBq6YiTq/bM= -github.com/hashicorp/vault/sdk v0.4.1 h1:3SaHOJY687jY1fnB61PtL0cOkKItphrbLmux7T92HBo= -github.com/hashicorp/vault/sdk v0.4.1/go.mod h1:aZ3fNuL5VNydQk8GcLJ2TV8YCRVvyaakYkhZRoVuhj0= +github.com/hashicorp/vault/api v1.14.0 h1:Ah3CFLixD5jmjusOgm8grfN9M0d+Y8fVR2SW0K6pJLU= +github.com/hashicorp/vault/api v1.14.0/go.mod h1:pV9YLxBGSz+cItFDd8Ii4G17waWOQ32zVjMWHe/cOqk= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 h1:xixZ2bWeofWV68J+x6AzmKuVM/JWCQwkWm6GW/MUR6I= @@ -862,7 +817,6 @@ github.com/infracost/terragrunt v0.47.1-0.20240501143558-4c01e72103df h1:aGlX662 github.com/infracost/terragrunt v0.47.1-0.20240501143558-4c01e72103df/go.mod h1:xmRpWI+M4KlZbMQp1pj20CdXlZf5eIkRND5ybNkdnus= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= @@ -879,9 +833,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22 github.com/joyent/triton-go v0.0.0-20180313100802-d8f9c0314926/go.mod h1:U+RSyWxWd04xTqnuOQxnai7XGS2PrPY2cfGoDKtMHjA= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -906,8 +858,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -936,7 +886,6 @@ github.com/masterzen/simplexml v0.0.0-20190410153822-31eea3082786/go.mod h1:kCEb github.com/masterzen/winrm v0.0.0-20200615185753-c42b5136ff88/go.mod h1:a2HXwefeat3evJHxFXSayvRHpYEPJYtErl4uIzfaUqY= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -991,9 +940,8 @@ github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/z github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= -github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/panicwrap v1.0.0 h1:67zIyVakCIvcs69A0FGfZjBdPleaonSgGlXRSRlb6fE= github.com/mitchellh/panicwrap v1.0.0/go.mod h1:pKvZHwWrZowLUzftuFq7coarnxbBXU4aQh3N0BJOeeA= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= @@ -1048,18 +996,12 @@ github.com/owenrumney/go-sarif v1.1.1 h1:QNObu6YX1igyFKhdzd7vgzmw7XsWN3/6NMGuDzB github.com/owenrumney/go-sarif v1.1.1/go.mod h1:dNDiPlF04ESR/6fHlPyq7gHKmrM0sHUvAGjsoh8ZH0U= github.com/packer-community/winrmcp v0.0.0-20180921211025-c76d91c1e7db/go.mod h1:f6Izs6JvFTdnRbziASagjZ2vmf55NSIkC/weStxCHqk= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= -github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= -github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/browser v0.0.0-20201207095918-0426ae3fba23/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1 h1:F++O52m40owAmADcojzM+9gyjmMOY/T4oYJkgFDH8RE= @@ -1072,25 +1014,18 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr github.com/posener/complete v1.2.1/go.mod h1:6gapUrK/U1TAN7ciCoNRIdVC5sbdBTUh1DKN0g6uH7E= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.13.1 h1:3gMjIY2+/hzmqhtUC/aQNYldJA6DtH3CgQvwS+02K1c= github.com/prometheus/client_golang v1.13.1/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= @@ -1196,7 +1131,6 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1 github.com/tombuildsstuff/giovanni v0.15.1/go.mod h1:0TZugJPEtqzPlMpuJHYfXY6Dq2uLPrXf98D2XQSxNbA= github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7 h1:qDMxFVd8Zo0rIhnEBdCIbR+T6WgjwkxpFZMN8zZmmjg= github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7/go.mod h1:5hzpfalEjfcJWp9yq75/EZoEu2Mzm34eJAPm3HOW2tw= -github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/ugorji/go v0.0.0-20180813092308-00b869d2f4a5/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= @@ -1266,8 +1200,6 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1287,7 +1219,6 @@ golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -1295,8 +1226,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1355,7 +1286,6 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1402,8 +1332,8 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1566,12 +1496,11 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1837,7 +1766,6 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= @@ -1885,9 +1813,6 @@ gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= -gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= @@ -1898,7 +1823,6 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 98337025d9c375fb71c200c970d6698779de37b1 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Thu, 5 Sep 2024 16:22:38 +0100 Subject: [PATCH 04/81] fix: indentation of generated config (#3201) --- internal/hcl/parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/hcl/parser.go b/internal/hcl/parser.go index 4f7d31a9427..c2f4c166410 100644 --- a/internal/hcl/parser.go +++ b/internal/hcl/parser.go @@ -349,7 +349,7 @@ func (p *Parser) YAML() string { str.WriteString(" skip_autodetect: true\n") if len(p.tfEnvVars) > 0 { - str.WriteString(" terraform_vars:\n") + str.WriteString(" terraform_vars:\n") keys := make([]string, 0, len(p.tfEnvVars)) for key := range p.tfEnvVars { From 7b67deb17fa5419c88f0c725d8b1026bad58d2e6 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Mon, 9 Sep 2024 14:22:12 +0100 Subject: [PATCH 05/81] enhance: improve Terragrunt performance for monorepos (#3196) * enhance: add support for downloading only subdirs of modules * chore: upgrade Terragrunt dep * enhance: use Terragrunt partial parse config cache * enhance: ensure any submodules and symlinks are checked out if sparse-checkout is used * chore: add option for enabling sparse checkout * chore: update terragrunt and go-getter refs * chore: add comments * chore: use trace logging * chore: update dep shas --- go.mod | 6 +- go.sum | 18 +- internal/hcl/modules/loader.go | 251 ++++++++++++++++++ .../terraform/terragrunt_hcl_provider.go | 56 +++- 4 files changed, 317 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 20b57e7095d..31f555e13b3 100644 --- a/go.mod +++ b/go.mod @@ -181,6 +181,7 @@ require ( github.com/hashicorp/vault/api v1.14.0 // indirect github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 // indirect github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef // indirect + github.com/huandu/go-clone v1.7.2 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jessevdk/go-flags v1.5.0 // indirect github.com/jstemmer/go-junit-report v1.0.0 // indirect @@ -264,9 +265,10 @@ replace github.com/jedib0t/go-pretty/v6 => github.com/aliscott/go-pretty/v6 v6.1 replace github.com/spf13/cobra => github.com/spf13/cobra v1.4.0 -//replace github.com/gruntwork-io/terragrunt => github.com/infracost/terragrunt v0.47.1-0.20231211141424-6a52de9a284a -replace github.com/gruntwork-io/terragrunt => github.com/infracost/terragrunt v0.47.1-0.20240501143558-4c01e72103df +replace github.com/gruntwork-io/terragrunt => github.com/infracost/terragrunt v0.47.1-0.20240909111416-82d45309175e replace github.com/heimdalr/dag => github.com/aliscott/dag v1.3.2-0.20231115114512-4ce18c825f94 replace github.com/shurcooL/graphql => github.com/spacelift-io/graphql v1.2.0 + +replace github.com/hashicorp/go-getter => github.com/infracost/go-getter v0.0.0-20240909111353-c0d2eebadfd5 diff --git a/go.sum b/go.sum index 00ec962b3d8..32aadf75100 100644 --- a/go.sum +++ b/go.sum @@ -313,7 +313,6 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= github.com/aws/aws-sdk-go v1.31.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.44.122 h1:p6mw01WBaNpbdP2xrisz5tIkcNwzj/HysobNoaAHjgo= @@ -720,9 +719,6 @@ github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9n github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-cty-funcs v0.0.0-20230405223818-a090f58aa992 h1:fYOrSfO5C9PmFGtmRWSYGqq52SOoE2dXMtAn2Xzh1LQ= github.com/hashicorp/go-cty-funcs v0.0.0-20230405223818-a090f58aa992/go.mod h1:Abjk0jbRkDaNCzsRhOv2iDCofYpX1eVsjozoiK63qLA= -github.com/hashicorp/go-getter v1.5.1/go.mod h1:a7z7NPPfNQpJWcn4rSWFtdrSldqLdLPEF3d8nFMsSLM= -github.com/hashicorp/go-getter v1.7.6 h1:5jHuM+aH373XNtXl9TNTUH5Qd69Trve11tHIrB+6yj4= -github.com/hashicorp/go-getter v1.7.6/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v0.15.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= @@ -760,7 +756,6 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= @@ -800,6 +795,10 @@ github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87/go.mod h1:CtWFDAQg github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef h1:A9HsByNhogrvm9cWb28sjiS3i7tcKCkflWFEkHfuAgM= github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef/go.mod h1:lADxMC39cJJqL93Duh1xhAs4I2Zs8mKS89XWXFGp9cs= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/go-assert v1.1.5 h1:fjemmA7sSfYHJD7CUqs9qTwwfdNAx7/j2/ZlHXzNB3c= +github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0JrPVhn/06U= +github.com/huandu/go-clone v1.7.2 h1:3+Aq0Ed8XK+zKkLjE2dfHg0XrpIfcohBE1K+c8Usxoo= +github.com/huandu/go-clone v1.7.2/go.mod h1:ReGivhG6op3GYr+UY3lS6mxjKp7MIGTknuU5TbTVaXE= github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= @@ -813,15 +812,16 @@ github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/infracost/terragrunt v0.47.1-0.20240501143558-4c01e72103df h1:aGlX6621I8AM6Rza0QorcAzABpYtMP9KV8yfgBX08VU= -github.com/infracost/terragrunt v0.47.1-0.20240501143558-4c01e72103df/go.mod h1:xmRpWI+M4KlZbMQp1pj20CdXlZf5eIkRND5ybNkdnus= +github.com/infracost/go-getter v0.0.0-20240909111353-c0d2eebadfd5 h1:FiK2b8h6CezRGI6CGs7YDVG9nbF2TQGMcRK9iSM37so= +github.com/infracost/go-getter v0.0.0-20240909111353-c0d2eebadfd5/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= +github.com/infracost/terragrunt v0.47.1-0.20240909111416-82d45309175e h1:YBOqpHRu4DHY5v9tyBR66PsdZOEL/Cs3kzhOuTbA6vM= +github.com/infracost/terragrunt v0.47.1-0.20240909111416-82d45309175e/go.mod h1:504J5iD5AjGgP5IwHkUWAcfoqvZIhrR1aEvyxDxX1VA= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -927,7 +927,6 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-linereader v0.0.0-20190213213312-1b945b3263eb/go.mod h1:OaY7UOoTkkrX3wRwjpYRKafIkkyeD0UtweSHAWWiqQM= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= @@ -1132,7 +1131,6 @@ github.com/tombuildsstuff/giovanni v0.15.1/go.mod h1:0TZugJPEtqzPlMpuJHYfXY6Dq2u github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7 h1:qDMxFVd8Zo0rIhnEBdCIbR+T6WgjwkxpFZMN8zZmmjg= github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7/go.mod h1:5hzpfalEjfcJWp9yq75/EZoEu2Mzm34eJAPm3HOW2tw= github.com/ugorji/go v0.0.0-20180813092308-00b869d2f4a5/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= -github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= diff --git a/internal/hcl/modules/loader.go b/internal/hcl/modules/loader.go index 47b07fb3931..598d9559928 100644 --- a/internal/hcl/modules/loader.go +++ b/internal/hcl/modules/loader.go @@ -1,11 +1,13 @@ package modules import ( + "bytes" "crypto/md5" //nolint "errors" "fmt" "net/url" "os" + "os/exec" "path" "path/filepath" "runtime" @@ -33,6 +35,12 @@ var ( tfManifestPath = ".terraform/modules/modules.json" supportedManifestVersion = "2.0" + + // maxSparseCheckoutDepth is the maximum depth to which we will follow symlinks when adding directories + // to the sparse-checkout file list. This is currently set to a low value, since increasing it is likely + // to cause performance issues. If we need to increase it in the future, we should consider adding an + // option to allow users to set this value. + maxSparseCheckoutDepth = 1 ) // ModuleLoader handles the loading of Terraform modules. It supports local, registry and other remote modules. @@ -279,6 +287,29 @@ func (m *ModuleLoader) loadModule(moduleCall *tfconfig.ModuleCall, parentPath st } m.logger.Debug().Msgf("loading local module %s from %s", key, dir) + + // If the module is a in a git repo we need to check that it has been checked out. + // If it hasn't been checked out then we need to add it to the sparse-checkout file list. + m.logger.Trace().Msgf("finding git repo root for path %s", parentPath) + repoRoot, err := findGitRepoRoot(parentPath) + if err == nil { + // Get the dir relative to the repoRoot + absDir, err := filepath.Abs(dir) + if err != nil { + return nil, err + } + + relDir, err := filepath.Rel(repoRoot, absDir) + if err != nil { + return nil, err + } + + err = m.checkoutPathIfRequired(repoRoot, relDir) + if err != nil { + return nil, err + } + } + return &ManifestModule{ Key: key, Source: source, @@ -315,6 +346,199 @@ func (m *ModuleLoader) loadModule(moduleCall *tfconfig.ModuleCall, parentPath st return manifestModule, nil } +// checkoutPathIfRequired checks if the given directories are in the sparse-checkout file list and adds them if not. +func (m *ModuleLoader) checkoutPathIfRequired(repoRoot string, dirs string) error { + // Lock the git repo root so we don't have multiple calls trying to read and update the sparse-checkout file list. + unlock := m.sync.Lock(repoRoot) + defer unlock() + + // Get the list of sparse checkout directories (and assume sparse checkout is enabled if this succeeds) + m.logger.Trace().Msgf("getting sparse checkout directories for path %s", repoRoot) + existingDirs, err := getSparseCheckoutDirs(repoRoot) + if err != nil { + // If the error indicates that sparse checkout is not enabled, just return nil + if err.Error() == "sparse-checkout not enabled" { + m.logger.Trace().Msgf("sparse-checkout not enabled for path %s", repoRoot) + return nil + } + return err + } + + mu := &sync.Mutex{} + + return RecursivelyAddDirsToSparseCheckout(repoRoot, existingDirs, []string{dirs}, mu, m.logger, 0) +} + +// RecursivelyAddDirsToSparseCheckout adds the given directories to the sparse-checkout file list. +// It then checks any symlinks within the directories and adds them to the sparse-checkout file list as well. +func RecursivelyAddDirsToSparseCheckout(repoRoot string, existingDirs []string, dirs []string, mu *sync.Mutex, logger zerolog.Logger, depth int) error { + var newDirs []string + + // Sort the existing directories and dirs to be added by length + // This ensures that parent directories are added before child directories + // since they cover the child directories anyway. + sort.Slice(existingDirs, func(i, j int) bool { + return len(existingDirs[i]) < len(existingDirs[j]) + }) + sort.Slice(dirs, func(i, j int) bool { + return len(dirs[i]) < len(dirs[j]) + }) + + for _, dir := range dirs { + if isCoveredByExistingDirs(existingDirs, dir) { + continue + } + + existingDirs = append(existingDirs, dir) + newDirs = append(newDirs, dir) + } + if len(newDirs) == 0 { + return nil + } + + logger.Trace().Msgf("adding dirs to sparse-checkout for repo %s: %v", repoRoot, newDirs) + mu.Lock() + err := setSparseCheckoutDirs(repoRoot, existingDirs) + mu.Unlock() + if err != nil { + return err + } + + if depth >= maxSparseCheckoutDepth { + return nil + } + + var additionalDirs []string + for _, dir := range newDirs { + symlinkedDirs, err := ResolveSymLinkedDirs(repoRoot, dir) + if err != nil { + return err + } + + for _, symlinkedDir := range symlinkedDirs { + if !isCoveredByExistingDirs(existingDirs, symlinkedDir) { + additionalDirs = append(additionalDirs, symlinkedDir) + } + } + } + + if len(additionalDirs) > 0 { + logger.Trace().Msgf("recursively adding symlinked dirs to sparse-checkout for repo %s: %v", repoRoot, additionalDirs) + return RecursivelyAddDirsToSparseCheckout(repoRoot, existingDirs, additionalDirs, mu, logger, depth+1) + } + + return nil +} + +// isCoveredByExistingDirs checks if the given directory is covered by any of the existing directories +// i.e. if it is a subdirectory of any of the existing directories. +func isCoveredByExistingDirs(existingDirs []string, dir string) bool { + for _, existingDir := range existingDirs { + if dir == existingDir || strings.HasPrefix(dir, existingDir+string(filepath.Separator)) { + return true + } + } + return false +} + +// findGitRepoRoot finds the root of the Git repository given a starting path +func findGitRepoRoot(startPath string) (string, error) { + cmd := exec.Command("git", "rev-parse", "--show-toplevel") + cmd.Dir = startPath + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("not a git repository") + } + return strings.TrimSpace(string(output)), nil +} + +// getSparseCheckoutDirs gets the list of directories currently in sparse-checkout +func getSparseCheckoutDirs(repoRoot string) ([]string, error) { + cmd := exec.Command("git", "sparse-checkout", "list") + cmd.Dir = repoRoot + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + stderrStr := strings.TrimSpace(stderr.String()) + if strings.Contains(stderrStr, "this worktree is not sparse") { + return nil, errors.New("sparse-checkout not enabled") + } + + return nil, fmt.Errorf("error getting sparse-checkout list: %w", err) + } + + output := strings.TrimSpace(stdout.String()) + if output == "" { + return nil, nil + } + + return strings.Split(output, "\n"), nil +} + +// setSparseCheckoutDirs sets the sparse-checkout list to include the given directories +func setSparseCheckoutDirs(repoRoot string, dirs []string) error { + cmd := exec.Command("git", "sparse-checkout", "set") + cmd.Dir = repoRoot + cmd.Args = append(cmd.Args, dirs...) + if err := cmd.Run(); err != nil { + return fmt.Errorf("error setting sparse-checkout list: %w", err) + } + + return nil +} + +// ResolveSymLinkedDirs traverses the directory to find symlinks and resolve their destinations +func ResolveSymLinkedDirs(repoRoot, dir string) ([]string, error) { + dirsToInclude := make(map[string]struct{}) + basePath := path.Join(repoRoot, dir) + + err := filepath.Walk(basePath, func(currentPath string, info os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("error walking the path: %w", err) + } + if info.Mode()&os.ModeSymlink != 0 { + resolvedPath, err := resolveSymlink(currentPath) + if err != nil { + return fmt.Errorf("error resolving symlink: %w", err) + } + + // Ensure the resolved path is within the repoRoot + if !strings.HasPrefix(resolvedPath, repoRoot) { + return nil + } + + resolvedDir := filepath.Dir(resolvedPath) + + resolvedRelDir, err := filepath.Rel(repoRoot, resolvedDir) + if err != nil { + return fmt.Errorf("error getting relative path: %w", err) + } + + // Skip if it's already included or within the original directory + if _, alreadyIncluded := dirsToInclude[resolvedRelDir]; alreadyIncluded || strings.HasPrefix(resolvedRelDir, dir) { + return nil + } + + dirsToInclude[resolvedRelDir] = struct{}{} + } + return nil + }) + + if err != nil { + return nil, err + } + + dirs := make([]string, 0, len(dirsToInclude)) + for d := range dirsToInclude { + dirs = append(dirs, d) + } + + return dirs, nil +} + func (m *ModuleLoader) loadModuleFromPath(fullPath string) (*tfconfig.Module, error) { mod := tfconfig.NewModule(fullPath) @@ -425,6 +649,19 @@ func (m *ModuleLoader) loadRemoteModule(key string, source string) (*ManifestMod return nil, err } + // If sparse checkout is enabled add the subdir to the module URL as a query param + // so go-getter only downloads the required directory. + if os.Getenv("INFRACOST_SPARSE_CHECKOUT") == "true" { + u, err := url.Parse(moduleAddr) + if err != nil { + return nil, err + } + q := u.Query() + q.Set("subdir", submodulePath) + u.RawQuery = q.Encode() + moduleAddr = u.String() + } + dest := m.downloadDest(moduleAddr, "") moduleDownloadDir, err := m.cachePathRel(dest) if err != nil { @@ -601,6 +838,20 @@ func mapSource(sourceMap config.TerraformSourceMap, source string) (SourceMapRes return result, nil } +// resolveSymlink resolves symlinks even if the target does not exist +func resolveSymlink(path string) (string, error) { + link, err := os.Readlink(path) + if err != nil { + return "", err + } + + if filepath.IsAbs(link) { + return link, nil + } + + return filepath.Join(filepath.Dir(path), link), nil +} + func getProcessCount() int { numWorkers := 4 numCPU := runtime.NumCPU() diff --git a/internal/providers/terraform/terragrunt_hcl_provider.go b/internal/providers/terraform/terragrunt_hcl_provider.go index 0d88dbaa8aa..78feb3a2ecc 100644 --- a/internal/providers/terraform/terragrunt_hcl_provider.go +++ b/internal/providers/terraform/terragrunt_hcl_provider.go @@ -426,6 +426,7 @@ func (p *TerragruntHCLProvider) prepWorkingDirs() ([]*terragruntWorkingDirInfo, Env: p.env, IgnoreExternalDependencies: true, SourceMap: p.ctx.RunContext.Config.TerraformSourceMap, + UsePartialParseConfigCache: true, RunTerragrunt: func(opts *tgoptions.TerragruntOptions) (err error) { defer func() { unexpectedErr := recover() @@ -563,7 +564,7 @@ func (p *TerragruntHCLProvider) runTerragrunt(opts *tgoptions.TerragruntOptions) return } if sourceURL != "" { - updatedWorkingDir, err := downloadSourceOnce(sourceURL, opts, terragruntConfig) + updatedWorkingDir, err := downloadSourceOnce(sourceURL, opts, terragruntConfig, p.logger) if err != nil { info.error = err @@ -633,8 +634,35 @@ func (p *TerragruntHCLProvider) runTerragrunt(opts *tgoptions.TerragruntOptions) return info } +func splitModuleSubDir(moduleSource string) (string, string, error) { + moduleAddr, submodulePath := getter.SourceDirSubdir(moduleSource) + if strings.HasPrefix(submodulePath, "../") { + return "", "", fmt.Errorf("invalid submodule path '%s'", submodulePath) + } + + return moduleAddr, submodulePath, nil +} + // downloadSourceOnce thread-safely makes sure the sourceURL is only downloaded once -func downloadSourceOnce(sourceURL string, opts *tgoptions.TerragruntOptions, terragruntConfig *tgconfig.TerragruntConfig) (string, error) { +func downloadSourceOnce(sourceURL string, opts *tgoptions.TerragruntOptions, terragruntConfig *tgconfig.TerragruntConfig, logger zerolog.Logger) (string, error) { + _, modAddr, err := splitModuleSubDir(sourceURL) + if err != nil { + return "", err + } + + // If sparse checkout is enabled add the subdir to the Source URL as a query param + // so go-getter only downloads the required directory. + if os.Getenv("INFRACOST_SPARSE_CHECKOUT") == "true" { + u, err := url.Parse(sourceURL) + if err != nil { + return "", err + } + q := u.Query() + q.Set("subdir", modAddr) + u.RawQuery = q.Encode() + sourceURL = u.String() + } + source, err := tfsource.NewSource(sourceURL, opts.DownloadDir, opts.WorkingDir, terragruntConfig.GenerateConfigs, opts.Logger) if err != nil { return "", err @@ -664,11 +692,35 @@ func downloadSourceOnce(sourceURL string, opts *tgoptions.TerragruntOptions, ter } } + if modAddr != "" && isGitDir(dir) { + symlinkedDirs, err := modules.ResolveSymLinkedDirs(dir, modAddr) + if err != nil { + return "", err + } + + if len(symlinkedDirs) > 0 { + mu := &sync.Mutex{} + logger.Trace().Msgf("recursively adding symlinked dirs to sparse-checkout for repo %s: %v", dir, symlinkedDirs) + // Using a depth of 1 here since the submodule directory is already downloaded, so only need + // to add the symlinked directories to the sparse-checkout. + err := modules.RecursivelyAddDirsToSparseCheckout(dir, []string{modAddr}, symlinkedDirs, mu, logger, 1) + if err != nil { + return "", err + } + } + } + terragruntDownloadedDirs.Store(dir, true) return source.WorkingDir, nil } +// isGitDir checks if the directory is a git directory +func isGitDir(dir string) bool { + _, err := os.Stat(filepath.Join(dir, ".git")) + return err == nil +} + func forceHttpsDownload(sourceURL string, opts *tgoptions.TerragruntOptions, terragruntConfig *tgconfig.TerragruntConfig) bool { newSource, err := getter.Detect(sourceURL, opts.WorkingDir, getter.Detectors) if err != nil { From ef46dc1fadfd0609a989c710ed1e66ab79fbc0c6 Mon Sep 17 00:00:00 2001 From: Tim McFadden <52185+tim775@users.noreply.github.com> Date: Thu, 12 Sep 2024 10:56:43 -0400 Subject: [PATCH 06/81] fix: save posted comment when cloud is enabled (#3204) --- cmd/infracost/comment.go | 7 ++++--- cmd/infracost/comment_github.go | 2 +- internal/apiclient/dashboard.go | 7 ++++++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/cmd/infracost/comment.go b/cmd/infracost/comment.go index c8f45437155..ffeab3bc6ac 100644 --- a/cmd/infracost/comment.go +++ b/cmd/infracost/comment.go @@ -119,9 +119,10 @@ func buildCommentOutput(cmd *cobra.Command, ctx *config.RunContext, paths []stri // the full comment markdown has been received from the API addRun or loaded from the comment-path file, // so use that instead of building the output using the output.ToMarkdown templates. out = &CommentOutput{ - Body: commentData, - HasDiff: combined.HasDiff(), - ValidAt: &combined.TimeGenerated, + Body: commentData, + HasDiff: combined.HasDiff(), + ValidAt: &combined.TimeGenerated, + AddRunResponse: result, } } diff --git a/cmd/infracost/comment_github.go b/cmd/infracost/comment_github.go index b4239be40bc..e3179739257 100644 --- a/cmd/infracost/comment_github.go +++ b/cmd/infracost/comment_github.go @@ -134,7 +134,7 @@ func commentGitHubCmd(ctx *config.RunContext) *cobra.Command { return err } - if res.Posted && ctx.IsCloudUploadExplicitlyEnabled() { + if res.Posted && ctx.IsCloudUploadEnabled() { dashboardClient := apiclient.NewDashboardAPIClient(ctx) if err := dashboardClient.SavePostedPrComment(ctx, commentOut.AddRunResponse.RunID, commentOut.Body); err != nil { logging.Logger.Err(err).Msg("could not save posted PR comment") diff --git a/internal/apiclient/dashboard.go b/internal/apiclient/dashboard.go index 4eb405af7b1..aebfd565561 100644 --- a/internal/apiclient/dashboard.go +++ b/internal/apiclient/dashboard.go @@ -124,10 +124,15 @@ func (c *DashboardAPIClient) SavePostedPrComment(ctx *config.RunContext, runId, q := `mutation SavePostedPrComment($runId: String!, $comment: String!) { savePostedPrComment(runId: $runId, comment: $comment) }` - _, err := c.DoQueries([]GraphQLQuery{{q, map[string]interface{}{"runId": runId, "comment": comment}}}) + results, err := c.DoQueries([]GraphQLQuery{{q, map[string]interface{}{"runId": runId, "comment": comment}}}) if err != nil { return err } + if len(results) > 0 { + if results[0].Get("errors").Exists() { + return errors.New(results[0].Get("errors").String()) + } + } return nil } From bb8edf6af93d7a56fbeafa27094483e27921aef3 Mon Sep 17 00:00:00 2001 From: Hugo Rut Date: Wed, 18 Sep 2024 10:46:11 +0200 Subject: [PATCH 07/81] refactor(autodect): exclude examples, include hidden dirs (#3205) Changes autodetect logic to exclude `examples` folders from autodetect. This is a common pattern for repos that are used as modules. Including the `examples` projects has lead to confusion and false positives with cost changes/policy failures. Therefore it has been decided that we should exclude these by default. Additionally, autodetect logic has been altered to include project from hidden folders. The initial logic (where we excluded hidden directories) was made as we falsely assumed users would not place valid Terraform in hidden directories. This has been proven wrong, so now we only exclude a handful of known hidden dirs (e.g. `.infracost`) from the project detection logic. I've also made a change to the `shouldUseProject` logic in the project detection algorithm so that users can use the `include_dirs` logic to override any of the changes above. E.g. if a user wanted to include the `examples` directory for a repository. --- ...kdown_terragrunt_diff_project_error.golden | 12 ++--- .../{example => exp}/dev/terragrunt.hcl | 0 .../{example => exp}/modules/example/main.tf | 0 .../{example => exp}/prod/main.tf | 0 .../{example => exp}/prod/provider.tf | 0 .../{example => exp}/prod/terragrunt.hcl | 0 .../{example => exp}/terragrunt.hcl | 0 .../prior.json | 48 +++++++++---------- .../excludes_examples/expected.golden | 7 +++ .../generate/excludes_examples/tree.txt | 11 +++++ .../includes_hidden_dirs/expected.golden | 10 ++++ .../generate/includes_hidden_dirs/tree.txt | 11 +++++ internal/hcl/project_locator.go | 43 +++++++++++++++-- internal/testutil/generate.go | 6 +++ 14 files changed, 114 insertions(+), 34 deletions(-) rename cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/{example => exp}/dev/terragrunt.hcl (100%) rename cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/{example => exp}/modules/example/main.tf (100%) rename cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/{example => exp}/prod/main.tf (100%) rename cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/{example => exp}/prod/provider.tf (100%) rename cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/{example => exp}/prod/terragrunt.hcl (100%) rename cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/{example => exp}/terragrunt.hcl (100%) create mode 100644 cmd/infracost/testdata/generate/excludes_examples/expected.golden create mode 100644 cmd/infracost/testdata/generate/excludes_examples/tree.txt create mode 100644 cmd/infracost/testdata/generate/includes_hidden_dirs/expected.golden create mode 100644 cmd/infracost/testdata/generate/includes_hidden_dirs/tree.txt diff --git a/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/breakdown_terragrunt_diff_project_error.golden b/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/breakdown_terragrunt_diff_project_error.golden index 599d3f7a318..88eea988e09 100644 --- a/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/breakdown_terragrunt_diff_project_error.golden +++ b/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/breakdown_terragrunt_diff_project_error.golden @@ -1,19 +1,19 @@ ────────────────────────────────── -Project: REPLACED_PROJECT_PATH/testdata/breakdown_terragrunt_diff_project_error/example/dev -Module path: example/dev +Project: REPLACED_PROJECT_PATH/testdata/breakdown_terragrunt_diff_project_error/exp/dev +Module path: exp/dev Errors: Error processing module at 'REPLACED_PROJECT_PATH/testdata/breakdown_terragrunt_diff_project_error/foo/terragrunt.hcl'. How this module was found: - dependency of module at 'REPLACED_PROJECT_PATH/testdata/breakdown_terragrunt_diff_project_error/example/dev'. Underlying error: + dependency of module at 'REPLACED_PROJECT_PATH/testdata/breakdown_terragrunt_diff_project_error/exp/dev'. Underlying error: Error reading file at path REPLACED_PROJECT_PATH/testdata/breakdown_terragrunt_diff_project_error/foo/terragrunt.hcl: open REPLACED_PROJECT_PATH/testdata/breakdown_terragrunt_diff_project_error/foo/terragrunt.hcl: no such file or directory ────────────────────────────────── -Project: REPLACED_PROJECT_PATH/testdata/breakdown_terragrunt_diff_project_error/example/prod -Module path: example/prod +Project: REPLACED_PROJECT_PATH/testdata/breakdown_terragrunt_diff_project_error/exp/prod +Module path: exp/prod Errors: Error processing module at 'REPLACED_PROJECT_PATH/testdata/breakdown_terragrunt_diff_project_error/foo/terragrunt.hcl'. How this module was found: - dependency of module at 'REPLACED_PROJECT_PATH/testdata/breakdown_terragrunt_diff_project_error/example/dev'. Underlying error: + dependency of module at 'REPLACED_PROJECT_PATH/testdata/breakdown_terragrunt_diff_project_error/exp/dev'. Underlying error: Error reading file at path REPLACED_PROJECT_PATH/testdata/breakdown_terragrunt_diff_project_error/foo/terragrunt.hcl: open REPLACED_PROJECT_PATH/testdata/breakdown_terragrunt_diff_project_error/foo/terragrunt.hcl: no such file or directory diff --git a/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/dev/terragrunt.hcl b/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/dev/terragrunt.hcl similarity index 100% rename from cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/dev/terragrunt.hcl rename to cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/dev/terragrunt.hcl diff --git a/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/modules/example/main.tf b/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/modules/example/main.tf similarity index 100% rename from cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/modules/example/main.tf rename to cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/modules/example/main.tf diff --git a/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/prod/main.tf b/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/prod/main.tf similarity index 100% rename from cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/prod/main.tf rename to cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/prod/main.tf diff --git a/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/prod/provider.tf b/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/prod/provider.tf similarity index 100% rename from cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/prod/provider.tf rename to cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/prod/provider.tf diff --git a/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/prod/terragrunt.hcl b/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/prod/terragrunt.hcl similarity index 100% rename from cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/prod/terragrunt.hcl rename to cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/prod/terragrunt.hcl diff --git a/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/terragrunt.hcl b/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/terragrunt.hcl similarity index 100% rename from cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/terragrunt.hcl rename to cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/terragrunt.hcl diff --git a/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/prior.json b/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/prior.json index 974f64a0954..5e8cf179caa 100644 --- a/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/prior.json +++ b/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/prior.json @@ -13,12 +13,12 @@ "currency": "USD", "projects": [ { - "name": "infracost/infracost/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/dev", + "name": "infracost/infracost/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/dev", "metadata": { - "path": "example/dev", + "path": "exp/dev", "type": "terragrunt_dir", - "terraformModulePath": "example/dev", - "vcsSubPath": "cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/dev" + "terraformModulePath": "exp/dev", + "vcsSubPath": "cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/dev" }, "pastBreakdown": { "resources": [ @@ -31,13 +31,13 @@ { "blockName": "aws_instance.web_app", "endLine": 40, - "filename": "example/.infracost/.terragrunt-cache/tb4lvRRn_XDlkIdcOMXrVOpwyZM-UE5g00fnu6Xl8bU4Czra1P4YRNg/modules/example/main.tf", + "filename": "exp/.infracost/.terragrunt-cache/tb4lvRRn_XDlkIdcOMXrVOpwyZM-UE5g00fnu6Xl8bU4Czra1P4YRNg/modules/exp/main.tf", "startLine": 26 } ], "checksum": "9d5686abb1504e7ea06a56c2c1997956424fb39d74039eee65d9fe33e848656e", "endLine": 40, - "filename": "example/.infracost/.terragrunt-cache/tb4lvRRn_XDlkIdcOMXrVOpwyZM-UE5g00fnu6Xl8bU4Czra1P4YRNg/modules/example/main.tf", + "filename": "exp/.infracost/.terragrunt-cache/tb4lvRRn_XDlkIdcOMXrVOpwyZM-UE5g00fnu6Xl8bU4Czra1P4YRNg/modules/exp/main.tf", "startLine": 26 }, "hourlyCost": "0.0711890410958904115", @@ -108,13 +108,13 @@ { "blockName": "aws_lambda_function.hello_world", "endLine": 48, - "filename": "example/.infracost/.terragrunt-cache/tb4lvRRn_XDlkIdcOMXrVOpwyZM-UE5g00fnu6Xl8bU4Czra1P4YRNg/modules/example/main.tf", + "filename": "exp/.infracost/.terragrunt-cache/tb4lvRRn_XDlkIdcOMXrVOpwyZM-UE5g00fnu6Xl8bU4Czra1P4YRNg/modules/exp/main.tf", "startLine": 42 } ], "checksum": "5b4d27384b5e9f180c9e4270fb612fe9ec2c0b34fd3aeb4ede827dbe21f4f142", "endLine": 48, - "filename": "example/.infracost/.terragrunt-cache/tb4lvRRn_XDlkIdcOMXrVOpwyZM-UE5g00fnu6Xl8bU4Czra1P4YRNg/modules/example/main.tf", + "filename": "exp/.infracost/.terragrunt-cache/tb4lvRRn_XDlkIdcOMXrVOpwyZM-UE5g00fnu6Xl8bU4Czra1P4YRNg/modules/exp/main.tf", "startLine": 42 }, "costComponents": [ @@ -162,13 +162,13 @@ { "blockName": "aws_instance.web_app", "endLine": 40, - "filename": "example/.infracost/.terragrunt-cache/tb4lvRRn_XDlkIdcOMXrVOpwyZM-UE5g00fnu6Xl8bU4Czra1P4YRNg/modules/example/main.tf", + "filename": "exp/.infracost/.terragrunt-cache/tb4lvRRn_XDlkIdcOMXrVOpwyZM-UE5g00fnu6Xl8bU4Czra1P4YRNg/modules/exp/main.tf", "startLine": 26 } ], "checksum": "9d5686abb1504e7ea06a56c2c1997956424fb39d74039eee65d9fe33e848656e", "endLine": 40, - "filename": "example/.infracost/.terragrunt-cache/tb4lvRRn_XDlkIdcOMXrVOpwyZM-UE5g00fnu6Xl8bU4Czra1P4YRNg/modules/example/main.tf", + "filename": "exp/.infracost/.terragrunt-cache/tb4lvRRn_XDlkIdcOMXrVOpwyZM-UE5g00fnu6Xl8bU4Czra1P4YRNg/modules/exp/main.tf", "startLine": 26 }, "hourlyCost": "0.0711890410958904115", @@ -239,13 +239,13 @@ { "blockName": "aws_lambda_function.hello_world", "endLine": 48, - "filename": "example/.infracost/.terragrunt-cache/tb4lvRRn_XDlkIdcOMXrVOpwyZM-UE5g00fnu6Xl8bU4Czra1P4YRNg/modules/example/main.tf", + "filename": "exp/.infracost/.terragrunt-cache/tb4lvRRn_XDlkIdcOMXrVOpwyZM-UE5g00fnu6Xl8bU4Czra1P4YRNg/modules/exp/main.tf", "startLine": 42 } ], "checksum": "5b4d27384b5e9f180c9e4270fb612fe9ec2c0b34fd3aeb4ede827dbe21f4f142", "endLine": 48, - "filename": "example/.infracost/.terragrunt-cache/tb4lvRRn_XDlkIdcOMXrVOpwyZM-UE5g00fnu6Xl8bU4Czra1P4YRNg/modules/example/main.tf", + "filename": "exp/.infracost/.terragrunt-cache/tb4lvRRn_XDlkIdcOMXrVOpwyZM-UE5g00fnu6Xl8bU4Czra1P4YRNg/modules/exp/main.tf", "startLine": 42 }, "costComponents": [ @@ -298,12 +298,12 @@ } }, { - "name": "infracost/infracost/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/prod", + "name": "infracost/infracost/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/prod", "metadata": { - "path": "example/prod", + "path": "exp/prod", "type": "terragrunt_dir", - "terraformModulePath": "example/prod", - "vcsSubPath": "cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/prod" + "terraformModulePath": "exp/prod", + "vcsSubPath": "cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/prod" }, "pastBreakdown": { "resources": [ @@ -316,13 +316,13 @@ { "blockName": "aws_instance.web_app", "endLine": 40, - "filename": "/Users/hugorut/code/infracost/infracost/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/prod/main.tf", + "filename": "/Users/hugorut/code/infracost/infracost/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/prod/main.tf", "startLine": 26 } ], "checksum": "68a408f4bf19420a57d0faab69d346b9c160280a7d3c172938837f9744a288c9", "endLine": 40, - "filename": "/Users/hugorut/code/infracost/infracost/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/prod/main.tf", + "filename": "/Users/hugorut/code/infracost/infracost/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/prod/main.tf", "startLine": 26 }, "hourlyCost": "1.024164383561643829", @@ -393,13 +393,13 @@ { "blockName": "aws_lambda_function.hello_world", "endLine": 48, - "filename": "/Users/hugorut/code/infracost/infracost/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/prod/main.tf", + "filename": "/Users/hugorut/code/infracost/infracost/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/prod/main.tf", "startLine": 42 } ], "checksum": "20f7d38c433778654686d7dfeba8bf2e446f110379bbe1adeb48f7ebe51b936e", "endLine": 48, - "filename": "/Users/hugorut/code/infracost/infracost/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/prod/main.tf", + "filename": "/Users/hugorut/code/infracost/infracost/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/prod/main.tf", "startLine": 42 }, "costComponents": [ @@ -447,13 +447,13 @@ { "blockName": "aws_instance.web_app", "endLine": 40, - "filename": "/Users/hugorut/code/infracost/infracost/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/prod/main.tf", + "filename": "/Users/hugorut/code/infracost/infracost/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/prod/main.tf", "startLine": 26 } ], "checksum": "68a408f4bf19420a57d0faab69d346b9c160280a7d3c172938837f9744a288c9", "endLine": 40, - "filename": "/Users/hugorut/code/infracost/infracost/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/prod/main.tf", + "filename": "/Users/hugorut/code/infracost/infracost/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/prod/main.tf", "startLine": 26 }, "hourlyCost": "1.024164383561643829", @@ -524,13 +524,13 @@ { "blockName": "aws_lambda_function.hello_world", "endLine": 48, - "filename": "/Users/hugorut/code/infracost/infracost/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/prod/main.tf", + "filename": "/Users/hugorut/code/infracost/infracost/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/prod/main.tf", "startLine": 42 } ], "checksum": "20f7d38c433778654686d7dfeba8bf2e446f110379bbe1adeb48f7ebe51b936e", "endLine": 48, - "filename": "/Users/hugorut/code/infracost/infracost/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/example/prod/main.tf", + "filename": "/Users/hugorut/code/infracost/infracost/cmd/infracost/testdata/breakdown_terragrunt_diff_project_error/exp/prod/main.tf", "startLine": 42 }, "costComponents": [ diff --git a/cmd/infracost/testdata/generate/excludes_examples/expected.golden b/cmd/infracost/testdata/generate/excludes_examples/expected.golden new file mode 100644 index 00000000000..ba9dea1d187 --- /dev/null +++ b/cmd/infracost/testdata/generate/excludes_examples/expected.golden @@ -0,0 +1,7 @@ +version: 0.1 + +projects: + - path: infra + name: infra + skip_autodetect: true + diff --git a/cmd/infracost/testdata/generate/excludes_examples/tree.txt b/cmd/infracost/testdata/generate/excludes_examples/tree.txt new file mode 100644 index 00000000000..9cdcf04e8fb --- /dev/null +++ b/cmd/infracost/testdata/generate/excludes_examples/tree.txt @@ -0,0 +1,11 @@ +. +├── examples +│ └── foo +│ └── main.tf +└── infra + ├── examples + │ ├── bar + │ │ └── main.tf + │ └── baz + │ └── main.tf + └── blank.tf diff --git a/cmd/infracost/testdata/generate/includes_hidden_dirs/expected.golden b/cmd/infracost/testdata/generate/includes_hidden_dirs/expected.golden new file mode 100644 index 00000000000..b25d44e62bf --- /dev/null +++ b/cmd/infracost/testdata/generate/includes_hidden_dirs/expected.golden @@ -0,0 +1,10 @@ +version: 0.1 + +projects: + - path: .infra/bat + name: .infra-bat + skip_autodetect: true + - path: .infra/baz + name: .infra-baz + skip_autodetect: true + diff --git a/cmd/infracost/testdata/generate/includes_hidden_dirs/tree.txt b/cmd/infracost/testdata/generate/includes_hidden_dirs/tree.txt new file mode 100644 index 00000000000..070ef13a9ce --- /dev/null +++ b/cmd/infracost/testdata/generate/includes_hidden_dirs/tree.txt @@ -0,0 +1,11 @@ +. +├── .infracost +│ └── foo +│ └── main.tf +├── .git +│ └── main.ft +└── .infra + ├── baz + │ └── main.tf + └── bat + └── main.tf diff --git a/internal/hcl/project_locator.go b/internal/hcl/project_locator.go index 1ee8d416e3f..ff5c9289d4c 100644 --- a/internal/hcl/project_locator.go +++ b/internal/hcl/project_locator.go @@ -65,6 +65,14 @@ var ( ".tfvars.json", ".auto.tfvars.json", } + defaultExcludedDirs = []string{ + ".infracost", + ".git", + ".terraform", + ".terragrunt-cache", + "example", + "examples", + } ) // EnvFileMatcher is used to match environment specific var files. @@ -121,6 +129,30 @@ func CreateEnvFileMatcher(names []string, extensions []string) *EnvFileMatcher { } } +func createWildcardGlobPaths(dirs []string) []string { + var paths []string + for _, dir := range dirs { + paths = append(paths, path.Join(dir, "**")) + } + + for _, dir := range dirs { + paths = append(paths, path.Join("**", path.Join(dir, "**"))) + } + + return paths +} + +func isDefaultExcluded(name string) bool { + for _, dir := range defaultExcludedDirs { + if dir == name { + return true + } + } + + return false + +} + // IsAutoVarFile checks if the var file is an auto.tfvars or terraform.tfvars. // These are special Terraform var files that are applied to every project // automatically. @@ -345,7 +377,7 @@ func NewProjectLocator(logger zerolog.Logger, config *ProjectLocatorConfig) *Pro modules: make(map[string]struct{}), moduleCalls: make(map[string][]string), discoveredVarFiles: make(map[string][]RootPathVarFile), - excludedDirs: config.ExcludedDirs, + excludedDirs: append(createWildcardGlobPaths(defaultExcludedDirs), config.ExcludedDirs...), changedObjects: config.ChangedObjects, includedDirs: config.IncludedDirs, pathOverrides: overrides, @@ -1295,8 +1327,10 @@ func (p *ProjectLocator) discoveredProjectsWithModulesFiltered() []discoveredPro } func (p *ProjectLocator) shouldUseProject(dir discoveredProject, force bool) bool { - if p.shouldSkipDir(dir.path) { - p.logger.Debug().Msgf("skipping directory %s as it is marked as excluded by --exclude-path", dir.path) + // if the directory is marked as excluded and not included we skip it. + // The include_dirs setting takes precedence over exclude_dirs. + if p.shouldSkipDir(dir.path) && !p.shouldIncludeDir(dir.path) { + p.logger.Trace().Msgf("skipping directory %s as it is marked as excluded by --exclude-path", dir.path) return false } @@ -1484,7 +1518,8 @@ func (p *ProjectLocator) walkPaths(fullPath string, level int, maxSearchDepth in if info.IsDir() { fullPath := filepath.Join(fullPath, info.Name()) - if strings.HasPrefix(info.Name(), ".") && !p.shouldIncludeDir(fullPath) { + // if the directory is one of the known excluded directories we skip it. + if isDefaultExcluded(info.Name()) && !p.shouldIncludeDir(fullPath) { continue } diff --git a/internal/testutil/generate.go b/internal/testutil/generate.go index 1494c8b217d..57baf95844f 100644 --- a/internal/testutil/generate.go +++ b/internal/testutil/generate.go @@ -92,6 +92,12 @@ pipeline { } } ` + case "blank.tf": + content = ` + variable "region" { + type = string + } + ` default: content = "" } From fe942a784b5632995d253e0d03ed7114b0e1227d Mon Sep 17 00:00:00 2001 From: Tim McFadden <52185+tim775@users.noreply.github.com> Date: Wed, 18 Sep 2024 07:49:42 -0400 Subject: [PATCH 08/81] fix: correct reference lookups of count/each resources in count/each modules (#3206) This fixes an issue where references for count or for each resources within a count or foreach module where not correctly matched because the reference lookup function incorrectly used the module index as the resource index. For example, a resource referencing module.mymod[0].aws_resource.my_resource[1] would mistakenly be mapped to module.mymod[0].aws_resource.my_resource[0]. --- internal/providers/terraform/parser.go | 4 +- internal/providers/terraform/parser_test.go | 80 +++++++++++++++++++++ 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/internal/providers/terraform/parser.go b/internal/providers/terraform/parser.go index f81a594ed68..9ec9c5cef02 100644 --- a/internal/providers/terraform/parser.go +++ b/internal/providers/terraform/parser.go @@ -1059,7 +1059,7 @@ func getModuleNames(addr string) []string { } func addressCountIndex(addr string) int { - r := regexp.MustCompile(`\[(\d+)\]`) + r := regexp.MustCompile(`\[(\d+)\]$`) m := r.FindStringSubmatch(addr) if len(m) > 0 { @@ -1072,7 +1072,7 @@ func addressCountIndex(addr string) int { } func addressKey(addr string) string { - r := regexp.MustCompile(`\["([^"]+)"\]`) + r := regexp.MustCompile(`\["([^"]+)"\]$`) m := r.FindStringSubmatch(addr) if len(m) > 0 { diff --git a/internal/providers/terraform/parser_test.go b/internal/providers/terraform/parser_test.go index e4e4e08e6eb..aeb62bc2d5e 100644 --- a/internal/providers/terraform/parser_test.go +++ b/internal/providers/terraform/parser_test.go @@ -985,6 +985,79 @@ func TestAddressModulePart(t *testing.T) { } } +func TestRemoveAddressCountIndex(t *testing.T) { + tests := []struct { + address string + expected int + }{ + {"aws_instance.my_instance", -1}, + {"data.aws_instance.my_instance", -1}, + {"aws_instance.my_instance[3]", 3}, + {"data.aws_instance.my_instance[3]", 3}, + {"aws_instance.my_instance[3]", 3}, + {"data.aws_instance.my_instance[3]", 3}, + // Modules + {"module.my_module.aws_instance.my_instance", -1}, + {"module.my_module.data.aws_instance.my_instance", -1}, + {"module.my_module.aws_instance.my_instance[3]", 3}, + {"module.my_module.data.aws_instance.my_instance[3]", 3}, + {"module.my_module.aws_instance.my_instance[3]", 3}, + {"module.my_module.data.aws_instance.my_instance[3]", 3}, + // Count modules + {"module.my_module[2].aws_instance.my_instance", -1}, + {"module.my_module[2].data.aws_instance.my_instance", -1}, + {"module.my_module[2].aws_instance.my_instance[3]", 3}, + {"module.my_module[2].data.aws_instance.my_instance[3]", 3}, + {"module.my_module[2].aws_instance.my_instance[3]", 3}, + {"module.my_module[2].data.aws_instance.my_instance[3]", 3}, + // Each resources + {"module.my_module[\"modindex.0\"].aws_instance.my_instance", -1}, + {"module.my_module[\"modindex.0\"].data.aws_instance.my_instance", -1}, + {"module.my_module[\"modindex.0\"].aws_instance.my_instance[\"index.1\"]", -1}, + {"module.my_module[\"modindex.0\"].data.aws_instance.my_instance[\"index.1\"]", -1}, + {"module.my_module[\"modindex.0\"].aws_instance.my_instance[\"index[1]\"]", -1}, + {"module.my_module[\"modindex.0\"].data.aws_instance.my_instance[\"index[1]\"]", -1}, + } + + for _, test := range tests { + actual := addressCountIndex(test.address) + assert.Equal(t, test.expected, actual) + } +} + +func TestAddressKey(t *testing.T) { + tests := []struct { + address string + expected string + }{ + {"aws_instance.my_instance", ""}, + {"data.aws_instance.my_instance", ""}, + {"aws_instance.my_instance[\"index.1\"]", "index.1"}, + {"data.aws_instance.my_instance[\"index.1\"]", "index.1"}, + {"aws_instance.my_instance[\"index[1]\"]", "index[1]"}, + {"data.aws_instance.my_instance[\"index[1]\"]", "index[1]"}, + // Modules + {"module.my_module.aws_instance.my_instance", ""}, + {"module.my_module.data.aws_instance.my_instance", ""}, + {"module.my_module.aws_instance.my_instance[\"index.1\"]", "index.1"}, + {"module.my_module.data.aws_instance.my_instance[\"index.1\"]", "index.1"}, + {"module.my_module.aws_instance.my_instance[\"index[1]\"]", "index[1]"}, + {"module.my_module.data.aws_instance.my_instance[\"index[1]\"]", "index[1]"}, + // Each modules + {"module.my_module[\"modindex.0\"].aws_instance.my_instance", ""}, + {"module.my_module[\"modindex.0\"].data.aws_instance.my_instance", ""}, + {"module.my_module[\"modindex.0\"].aws_instance.my_instance[\"index.1\"]", "index.1"}, + {"module.my_module[\"modindex.0\"].data.aws_instance.my_instance[\"index.1\"]", "index.1"}, + {"module.my_module[\"modindex.0\"].aws_instance.my_instance[\"index[1]\"]", "index[1]"}, + {"module.my_module[\"modindex.0\"].data.aws_instance.my_instance[\"index[1]\"]", "index[1]"}, + } + + for _, test := range tests { + actual := addressKey(test.address) + assert.Equal(t, test.expected, actual) + } +} + func TestRemoveAddressArrayPart(t *testing.T) { tests := []struct { address string @@ -1003,6 +1076,13 @@ func TestRemoveAddressArrayPart(t *testing.T) { {"module.my_module.data.aws_instance.my_instance[\"index.1\"]", "data.aws_instance.my_instance"}, {"module.my_module.aws_instance.my_instance[\"index[1]\"]", "aws_instance.my_instance"}, {"module.my_module.data.aws_instance.my_instance[\"index[1]\"]", "data.aws_instance.my_instance"}, + // Each modules + {"module.my_module[\"modindex.0\"].aws_instance.my_instance", "aws_instance.my_instance"}, + {"module.my_module[\"modindex.0\"].data.aws_instance.my_instance", "data.aws_instance.my_instance"}, + {"module.my_module[\"modindex.0\"].aws_instance.my_instance[\"index.1\"]", "aws_instance.my_instance"}, + {"module.my_module[\"modindex.0\"].data.aws_instance.my_instance[\"index.1\"]", "data.aws_instance.my_instance"}, + {"module.my_module[\"modindex.0\"].aws_instance.my_instance[\"index[1]\"]", "aws_instance.my_instance"}, + {"module.my_module[\"modindex.0\"].data.aws_instance.my_instance[\"index[1]\"]", "data.aws_instance.my_instance"}, } for _, test := range tests { From 8690d191038040d08aba59e87e65c94ba12d2f4e Mon Sep 17 00:00:00 2001 From: Tim McFadden <52185+tim775@users.noreply.github.com> Date: Thu, 19 Sep 2024 06:40:20 -0400 Subject: [PATCH 09/81] fix: do not create duplicates in references (#3208) --- internal/providers/terraform/parser.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/providers/terraform/parser.go b/internal/providers/terraform/parser.go index 9ec9c5cef02..b2120569502 100644 --- a/internal/providers/terraform/parser.go +++ b/internal/providers/terraform/parser.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "regexp" + "slices" "sort" "strconv" "strings" @@ -696,6 +697,9 @@ func (p *Parser) parseReferences(resData map[string]*schema.ResourceData, confLo if _, ok := idMap[defaultID]; !ok { idMap[defaultID] = []*schema.ResourceData{} } + if slices.Contains(idMap[defaultID], d) { + continue + } idMap[defaultID] = append(idMap[defaultID], d) } } @@ -706,6 +710,9 @@ func (p *Parser) parseReferences(resData map[string]*schema.ResourceData, confLo if _, ok := idMap[customID]; !ok { idMap[customID] = []*schema.ResourceData{} } + if slices.Contains(idMap[customID], d) { + continue + } idMap[customID] = append(idMap[customID], d) } } From 727d26cfaca974e82e3bcb774a222ab08c08ace8 Mon Sep 17 00:00:00 2001 From: Hugo Rut Date: Mon, 23 Sep 2024 14:14:59 +0200 Subject: [PATCH 10/81] chore: bump terragrunt version (#3210) Bumps terragrunt pkg version so that we pull in the decoding panic changes outlined in https://github.com/infracost/terragrunt/pull/17 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 31f555e13b3..71c7c0fa4b0 100644 --- a/go.mod +++ b/go.mod @@ -265,7 +265,7 @@ replace github.com/jedib0t/go-pretty/v6 => github.com/aliscott/go-pretty/v6 v6.1 replace github.com/spf13/cobra => github.com/spf13/cobra v1.4.0 -replace github.com/gruntwork-io/terragrunt => github.com/infracost/terragrunt v0.47.1-0.20240909111416-82d45309175e +replace github.com/gruntwork-io/terragrunt => github.com/infracost/terragrunt v0.47.1-0.20240920091357-980f132ffa5a replace github.com/heimdalr/dag => github.com/aliscott/dag v1.3.2-0.20231115114512-4ce18c825f94 diff --git a/go.sum b/go.sum index 32aadf75100..4e2112210b9 100644 --- a/go.sum +++ b/go.sum @@ -814,8 +814,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/infracost/go-getter v0.0.0-20240909111353-c0d2eebadfd5 h1:FiK2b8h6CezRGI6CGs7YDVG9nbF2TQGMcRK9iSM37so= github.com/infracost/go-getter v0.0.0-20240909111353-c0d2eebadfd5/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= -github.com/infracost/terragrunt v0.47.1-0.20240909111416-82d45309175e h1:YBOqpHRu4DHY5v9tyBR66PsdZOEL/Cs3kzhOuTbA6vM= -github.com/infracost/terragrunt v0.47.1-0.20240909111416-82d45309175e/go.mod h1:504J5iD5AjGgP5IwHkUWAcfoqvZIhrR1aEvyxDxX1VA= +github.com/infracost/terragrunt v0.47.1-0.20240920091357-980f132ffa5a h1:S24y2y7hdsgtT4HD/dUAiangWfPv4cz+n3f4/J2di0M= +github.com/infracost/terragrunt v0.47.1-0.20240920091357-980f132ffa5a/go.mod h1:504J5iD5AjGgP5IwHkUWAcfoqvZIhrR1aEvyxDxX1VA= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= From 0ee89cff8a0944afd06fbfe90d7a98b8a947dce4 Mon Sep 17 00:00:00 2001 From: Liam Galvin Date: Wed, 25 Sep 2024 11:20:29 +0100 Subject: [PATCH 11/81] feat: Add ap-southeast-5 aws region (#3212) * feat: Add ap-southeast-5 aws region --- internal/hcl/zones_aws.go | 6 ++++++ .../data_transfer_test/data_transfer_test.golden | 16 ++++++++++++---- .../data_transfer_test.usage.yml | 6 ++++++ .../resources/aws/cloudfront_distribution.go | 1 + .../aws/global_accelerator_endpoint_group.go | 1 + internal/resources/aws/util.go | 3 +++ 6 files changed, 29 insertions(+), 4 deletions(-) diff --git a/internal/hcl/zones_aws.go b/internal/hcl/zones_aws.go index 0c7fc4c3fc0..6e849e9887e 100644 --- a/internal/hcl/zones_aws.go +++ b/internal/hcl/zones_aws.go @@ -69,6 +69,12 @@ var awsZones = map[string]cty.Value{ "zone_ids": cty.ListVal([]cty.Value{cty.StringVal("apse4-az1"), cty.StringVal("apse4-az2"), cty.StringVal("apse4-az3")}), "group_names": cty.ListVal([]cty.Value{cty.StringVal("ap-southeast-4"), cty.StringVal("ap-southeast-4"), cty.StringVal("ap-southeast-4")}), }), + "ap-southeast-5": cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal("ap-southeast-5"), + "names": cty.ListVal([]cty.Value{cty.StringVal("ap-southeast-5a"), cty.StringVal("ap-southeast-5b"), cty.StringVal("ap-southeast-5c")}), + "zone_ids": cty.ListVal([]cty.Value{cty.StringVal("apse5-az1"), cty.StringVal("apse5-az2"), cty.StringVal("apse5-az3")}), + "group_names": cty.ListVal([]cty.Value{cty.StringVal("ap-southeast-5"), cty.StringVal("ap-southeast-5"), cty.StringVal("ap-southeast-5")}), + }), "ca-central-1": cty.ObjectVal(map[string]cty.Value{ "id": cty.StringVal("ca-central-1"), "names": cty.ListVal([]cty.Value{cty.StringVal("ca-central-1a"), cty.StringVal("ca-central-1b"), cty.StringVal("ca-central-1d"), cty.StringVal("ca-central-1-wl1-yto-wlz-1")}), diff --git a/internal/providers/terraform/aws/testdata/data_transfer_test/data_transfer_test.golden b/internal/providers/terraform/aws/testdata/data_transfer_test/data_transfer_test.golden index 03a42d7d0ab..9845b6402a3 100644 --- a/internal/providers/terraform/aws/testdata/data_transfer_test/data_transfer_test.golden +++ b/internal/providers/terraform/aws/testdata/data_transfer_test/data_transfer_test.golden @@ -113,6 +113,14 @@ ├─ Outbound data transfer to Internet (over 150TB) 3,400 GB $187.00 * └─ Outbound data transfer to other regions 750 GB $60.00 * + aws_data_transfer.ap-southeast-5 + ├─ Intra-region data transfer 2,000 GB $20.00 * + ├─ Outbound data transfer to Internet (first 10TB) 10,240 GB $1,105.92 * + ├─ Outbound data transfer to Internet (next 40TB) 40,960 GB $3,133.44 * + ├─ Outbound data transfer to Internet (next 100TB) 102,400 GB $7,557.12 * + ├─ Outbound data transfer to Internet (over 150TB) 3,400 GB $244.80 * + └─ Outbound data transfer to other regions 750 GB $60.00 * + aws_data_transfer.us-east-2 ├─ Intra-region data transfer 2,000 GB $20.00 * ├─ Outbound data transfer to Internet (first 10TB) 10,240 GB $921.60 * @@ -218,16 +226,16 @@ ├─ Outbound data transfer to US East regions 500 GB $5.00 * └─ Outbound data transfer to other regions 750 GB $15.00 * - OVERALL TOTAL $363,014.51 + OVERALL TOTAL $375,135.79 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. ────────────────────────────────── -27 cloud resources were detected: -∙ 27 were estimated +28 cloud resources were detected: +∙ 28 were estimated ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $0.00 ┃ $363,015 ┃ $363,015 ┃ +┃ main ┃ $0.00 ┃ $375,136 ┃ $375,136 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/aws/testdata/data_transfer_test/data_transfer_test.usage.yml b/internal/providers/terraform/aws/testdata/data_transfer_test/data_transfer_test.usage.yml index 9a433292df2..33416cb3e75 100644 --- a/internal/providers/terraform/aws/testdata/data_transfer_test/data_transfer_test.usage.yml +++ b/internal/providers/terraform/aws/testdata/data_transfer_test/data_transfer_test.usage.yml @@ -179,3 +179,9 @@ resource_usage: monthly_intra_region_gb: 1000 monthly_outbound_other_regions_gb: 750 monthly_outbound_internet_gb: 157000 + + aws_data_transfer.ap-southeast-5: + region: ap-southeast-5 + monthly_intra_region_gb: 1000 + monthly_outbound_other_regions_gb: 750 + monthly_outbound_internet_gb: 157000 diff --git a/internal/resources/aws/cloudfront_distribution.go b/internal/resources/aws/cloudfront_distribution.go index 6f3dcdf53d2..9b2f135d14a 100644 --- a/internal/resources/aws/cloudfront_distribution.go +++ b/internal/resources/aws/cloudfront_distribution.go @@ -458,6 +458,7 @@ var regionShieldMapping = map[string]string{ "ap-southeast-2": "australia", "ap-southeast-3": "indonesia", "ap-southeast-4": "australia", + "ap-southeast-5": "malaysia", "ap-south-1": "india", "ap-south-2": "india", "sa-east-1": "south_america", diff --git a/internal/resources/aws/global_accelerator_endpoint_group.go b/internal/resources/aws/global_accelerator_endpoint_group.go index 2e64611309b..2ff2375f15a 100644 --- a/internal/resources/aws/global_accelerator_endpoint_group.go +++ b/internal/resources/aws/global_accelerator_endpoint_group.go @@ -60,6 +60,7 @@ var regionCodeMapping = map[string]string{ "ap-southeast-2": "AP", "ap-southeast-3": "AP", "ap-southeast-4": "AP", + "ap-southeast-5": "AP", "ap-south-1": "AP", "ap-south-2": "AP", "me-central-1": "ME", diff --git a/internal/resources/aws/util.go b/internal/resources/aws/util.go index fb98a984697..2274542ff86 100644 --- a/internal/resources/aws/util.go +++ b/internal/resources/aws/util.go @@ -136,6 +136,7 @@ var RegionMapping = map[string]string{ "ap-southeast-2": "Asia Pacific (Sydney)", "ap-southeast-3": "Asia Pacific (Jakarta)", "ap-southeast-4": "Asia Pacific (Melbourne)", + "ap-southeast-5": "Asia Pacific (Malaysia)", "ap-south-1": "Asia Pacific (Mumbai)", "ap-south-2": "Asia Pacific (Hyderabad)", "me-central-1": "Middle East (UAE)", @@ -183,6 +184,7 @@ type RegionsUsage struct { EUSouth1 *float64 `infracost_usage:"eu_south_1"` EUWest3 *float64 `infracost_usage:"eu_west_3"` EUNorth1 *float64 `infracost_usage:"eu_north_1"` + ILCentral1 *float64 `infracost_usage:"il_central_1"` APEast1 *float64 `infracost_usage:"ap_east_1"` APNortheast1 *float64 `infracost_usage:"ap_northeast_1"` APNortheast2 *float64 `infracost_usage:"ap_northeast_2"` @@ -191,6 +193,7 @@ type RegionsUsage struct { APSoutheast2 *float64 `infracost_usage:"ap_southeast_2"` APSoutheast3 *float64 `infracost_usage:"ap_southeast_3"` APSoutheast4 *float64 `infracost_usage:"ap_southeast_4"` + APSoutheast5 *float64 `infracost_usage:"ap_southeast_5"` APSouth1 *float64 `infracost_usage:"ap_south_1"` MESouth1 *float64 `infracost_usage:"me_south_1"` SAEast1 *float64 `infracost_usage:"sa_east_1"` From 5a101e942257799f515a4d9704af48577d7dc1c1 Mon Sep 17 00:00:00 2001 From: Liam Galvin Date: Tue, 1 Oct 2024 16:12:16 +0100 Subject: [PATCH 12/81] enhance: Detect aws_s3_bucket_intelligent_tiering_configuration for use in lifecycle policy (#3217) * enhance: Detect aws_s3_bucket_intelligent_tiering_configuration for use in lifecycle policy --- ...wn_format_json_with_tags_aft_module.golden | 4 +++ internal/providers/terraform/aws/registry.go | 2 +- ...ucket_intelligent_tiering_configuration.go | 25 +++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 internal/providers/terraform/aws/s3_bucket_intelligent_tiering_configuration.go diff --git a/cmd/infracost/testdata/breakdown_format_json_with_tags_aft_module/breakdown_format_json_with_tags_aft_module.golden b/cmd/infracost/testdata/breakdown_format_json_with_tags_aft_module/breakdown_format_json_with_tags_aft_module.golden index 128ae11e45d..453bbde9f36 100644 --- a/cmd/infracost/testdata/breakdown_format_json_with_tags_aft_module/breakdown_format_json_with_tags_aft_module.golden +++ b/cmd/infracost/testdata/breakdown_format_json_with_tags_aft_module/breakdown_format_json_with_tags_aft_module.golden @@ -2032,3 +2032,7 @@ } Err: + +Logs: +WARN 2 aws_sns_topic prices missing across 2 resources + diff --git a/internal/providers/terraform/aws/registry.go b/internal/providers/terraform/aws/registry.go index 8eea2ff84c4..6af75c741a9 100644 --- a/internal/providers/terraform/aws/registry.go +++ b/internal/providers/terraform/aws/registry.go @@ -85,6 +85,7 @@ var ResourceRegistry []*schema.RegistryItem = []*schema.RegistryItem{ getRoute53ZoneRegistryItem(), getS3BucketAnalyticsConfigurationRegistryItem(), getS3BucketInventoryRegistryItem(), + getS3BucketIntelligentTieringConfigurationRegistryItem(), getS3BucketLifecycleConfigurationRegistryItem(), getS3BucketRegistryItem(), getS3BucketVersioningRegistryItem(), @@ -530,7 +531,6 @@ var FreeResources = []string{ "aws_s3_account_public_access_block", "aws_s3_bucket_acl", "aws_s3_bucket_cors_configuration", - "aws_s3_bucket_intelligent_tiering_configuration", "aws_s3_bucket_logging", "aws_s3_bucket_metric", "aws_s3_bucket_notification", diff --git a/internal/providers/terraform/aws/s3_bucket_intelligent_tiering_configuration.go b/internal/providers/terraform/aws/s3_bucket_intelligent_tiering_configuration.go new file mode 100644 index 00000000000..f7b6b678d1d --- /dev/null +++ b/internal/providers/terraform/aws/s3_bucket_intelligent_tiering_configuration.go @@ -0,0 +1,25 @@ +package aws + +import ( + "github.com/infracost/infracost/internal/schema" +) + +func getS3BucketIntelligentTieringConfigurationRegistryItem() *schema.RegistryItem { + return &schema.RegistryItem{ + Name: "aws_s3_bucket_intelligent_tiering_configuration", + RFunc: NewS3BucketIntelligentTieringConfiguration, + ReferenceAttributes: []string{"bucket"}, + } +} + +func NewS3BucketIntelligentTieringConfiguration(d *schema.ResourceData, u *schema.UsageData) *schema.Resource { + return &schema.Resource{ + Name: d.Address, + ResourceType: d.Type, + Tags: d.Tags, + DefaultTags: d.DefaultTags, + IsSkipped: true, + NoPrice: true, + SkipMessage: "Free resource.", + } +} From d587ab34bc72589f236fcb49b0eacc083e2ec2dd Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Thu, 10 Oct 2024 14:22:37 +0100 Subject: [PATCH 13/81] fix: Terragrunt dependency outputs mocks when functions are called --- .../terraform/terragrunt_hcl_provider.go | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/internal/providers/terraform/terragrunt_hcl_provider.go b/internal/providers/terraform/terragrunt_hcl_provider.go index 78feb3a2ecc..d54787733d0 100644 --- a/internal/providers/terraform/terragrunt_hcl_provider.go +++ b/internal/providers/terraform/terragrunt_hcl_provider.go @@ -770,7 +770,20 @@ func convertToCtyWithJson(val interface{}) (cty.Value, error) { } var ( - depRegexp = regexp.MustCompile(`dependency\.[\w\-.\[\]"]+`) + // depRegexp is used to find dependency references in Terragrunt files, e.g. dependency.foo.bar + // so that we can mock the outputs of these dependencies into a "shape" that is expected by any + // inputs that reference it. + // The following regex supports formats like: + // - dependency.foo.bar + // - dependency.foo.bar[0] + // - dependency.foo.bar["baz"] + // - dependency.foo.bar[0]["baz"] + // - tolist(dependency.foo.bar) + // - tolist(dependency.foo.bar)[0] + // - tomap(dependency.foo.bar)["baz"] + // - tolist(dependency.foo.bar["baz")[0] + // - tomap(dependency.foo.bar[0])["baz"] + depRegexp = regexp.MustCompile(`(?:\w+\()?dependency\.[\w\-.\[\]"]+(?:\)[\w\-.\[\]"]+)?(?:\))?`) indexRegexp = regexp.MustCompile(`(\w+)\[(\d+)]`) mapRegexp = regexp.MustCompile(`\["([\w\d]+)"]`) ) @@ -820,13 +833,21 @@ func (p *TerragruntHCLProvider) fetchDependencyOutputs(opts *tgoptions.Terragrun valueMap := moduleOutputs.AsValueMap() for _, match := range matches { - pieces := strings.Split(match, ".") + stripped := stripFunctionCalls(match) + pieces := strings.Split(stripped, ".") valueMap = mergeObjectWithDependencyMap(valueMap, pieces[1:]) } return cty.ObjectVal(valueMap) } +func stripFunctionCalls(input string) string { + re := regexp.MustCompile(`\w+\(([^)]+)\)`) + stripped := re.ReplaceAllString(input, "$1") + + return stripped +} + func mergeObjectWithDependencyMap(valueMap map[string]cty.Value, pieces []string) map[string]cty.Value { if valueMap == nil { valueMap = make(map[string]cty.Value) From f0523b012d44a0b2b03935217ba75a8bc0174041 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Thu, 10 Oct 2024 15:27:21 +0100 Subject: [PATCH 14/81] fix: merge any objects when new keys are found --- .../terraform/terragrunt_hcl_provider.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/providers/terraform/terragrunt_hcl_provider.go b/internal/providers/terraform/terragrunt_hcl_provider.go index d54787733d0..49ad818cd26 100644 --- a/internal/providers/terraform/terragrunt_hcl_provider.go +++ b/internal/providers/terraform/terragrunt_hcl_provider.go @@ -954,21 +954,21 @@ func mergeListWithDependencyMap(valueMap map[string]cty.Value, pieces []string, mockValue := cty.ObjectVal(mergeObjectWithDependencyMap(map[string]cty.Value{}, pieces[1:])) if v, ok := valueMap[key]; ok && isList(v) { - // if we have the index already in the dependency output, and it is known use the existing value. - // If the value is unknown we need to override it wil a mock as Terragrunt will explode when they - // try and marshal the cty values to JSON. - if v.HasIndex(indexVal).True() && v.Index(indexVal).IsKnown() { - return valueMap - } - existing := v.AsValueSlice() vals := make([]cty.Value, index+1) for i, value := range existing { - if value.IsKnown() { + if i != index && value.IsKnown() { vals[i] = value continue } + // if we are at the index and the value is known, and it is an object we need to merge the object + // with mock values for the rest of the pieces. + if i == index && value.IsKnown() && value.CanIterateElements() && !isList(value) { + vals[i] = cty.ObjectVal(mergeObjectWithDependencyMap(value.AsValueMap(), pieces[1:])) + continue + } + vals[i] = mockValue } From a464398821eda355aef8f053c8edae6c869d1ba6 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Thu, 10 Oct 2024 17:17:13 +0100 Subject: [PATCH 15/81] fix: always check if sparse checkout is enabled We can't rely on just checking the list, since it may still return a list even if sparse checkout is disabled --- internal/hcl/modules/loader.go | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/internal/hcl/modules/loader.go b/internal/hcl/modules/loader.go index 598d9559928..9ce13eac461 100644 --- a/internal/hcl/modules/loader.go +++ b/internal/hcl/modules/loader.go @@ -288,7 +288,7 @@ func (m *ModuleLoader) loadModule(moduleCall *tfconfig.ModuleCall, parentPath st m.logger.Debug().Msgf("loading local module %s from %s", key, dir) - // If the module is a in a git repo we need to check that it has been checked out. + // If the module is in a git repo we need to check that it has been checked out. // If it hasn't been checked out then we need to add it to the sparse-checkout file list. m.logger.Trace().Msgf("finding git repo root for path %s", parentPath) repoRoot, err := findGitRepoRoot(parentPath) @@ -352,6 +352,17 @@ func (m *ModuleLoader) checkoutPathIfRequired(repoRoot string, dirs string) erro unlock := m.sync.Lock(repoRoot) defer unlock() + // Check if sparse-checkout is enabled for the repo + enabled, err := isSparseCheckoutEnabled(repoRoot) + if err != nil { + return err + } + + if !enabled { + m.logger.Trace().Msgf("sparse-checkout not enabled for path %s", repoRoot) + return nil + } + // Get the list of sparse checkout directories (and assume sparse checkout is enabled if this succeeds) m.logger.Trace().Msgf("getting sparse checkout directories for path %s", repoRoot) existingDirs, err := getSparseCheckoutDirs(repoRoot) @@ -452,6 +463,23 @@ func findGitRepoRoot(startPath string) (string, error) { return strings.TrimSpace(string(output)), nil } +// isSparseCheckoutEnabled checks if sparse-checkout is enabled in the repository +func isSparseCheckoutEnabled(repoRoot string) (bool, error) { + cmd := exec.Command("git", "config", "--get", "core.sparseCheckout") + cmd.Dir = repoRoot + output, err := cmd.Output() + if err != nil { + // if exit status is 1, then the config is not set + var exitErr *exec.ExitError + if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 { + return false, nil + } + + return false, fmt.Errorf("error checking if sparse-checkout is enabled: %w", err) + } + return string(output) == "true\n", nil +} + // getSparseCheckoutDirs gets the list of directories currently in sparse-checkout func getSparseCheckoutDirs(repoRoot string) ([]string, error) { cmd := exec.Command("git", "sparse-checkout", "list") From 3066d56c8f6edd993d065aee0bd5d3c26c91ae5e Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Thu, 10 Oct 2024 17:18:09 +0100 Subject: [PATCH 16/81] chore: update module test line numbers --- .../expected.json | 96 +++++++++---------- .../expected.json | 96 +++++++++---------- .../renders_module_resources/expected.json | 24 +++-- 3 files changed, 114 insertions(+), 102 deletions(-) diff --git a/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module/expected.json b/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module/expected.json index 5217ada9097..083b0844e43 100644 --- a/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module/expected.json +++ b/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module/expected.json @@ -103,15 +103,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 923, - "endLine": 938 + "startLine": 924, + "endLine": 939 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 938, + "endLine": 939, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 923 + "startLine": 924 } }, { @@ -139,15 +139,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 912, - "endLine": 921 + "startLine": 913, + "endLine": 922 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 921, + "endLine": 922, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 912 + "startLine": 913 } }, { @@ -291,15 +291,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 923, - "endLine": 938 + "startLine": 924, + "endLine": 939 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 938, + "endLine": 939, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 923 + "startLine": 924 } }, { @@ -327,15 +327,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 912, - "endLine": 921 + "startLine": 913, + "endLine": 922 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 921, + "endLine": 922, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 912 + "startLine": 913 } }, { @@ -544,15 +544,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 923, - "endLine": 938 + "startLine": 924, + "endLine": 939 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 938, + "endLine": 939, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 923 + "startLine": 924 } }, { @@ -580,15 +580,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 912, - "endLine": 921 + "startLine": 913, + "endLine": 922 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 921, + "endLine": 922, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 912 + "startLine": 913 } }, { @@ -927,15 +927,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 923, - "endLine": 938 + "startLine": 924, + "endLine": 939 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 938, + "endLine": 939, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 923 + "startLine": 924 } }, { @@ -963,15 +963,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 912, - "endLine": 921 + "startLine": 913, + "endLine": 922 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 921, + "endLine": 922, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 912 + "startLine": 913 } }, { @@ -1115,15 +1115,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 923, - "endLine": 938 + "startLine": 924, + "endLine": 939 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 938, + "endLine": 939, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 923 + "startLine": 924 } }, { @@ -1151,15 +1151,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 912, - "endLine": 921 + "startLine": 913, + "endLine": 922 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 921, + "endLine": 922, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 912 + "startLine": 913 } }, { @@ -1368,15 +1368,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 923, - "endLine": 938 + "startLine": 924, + "endLine": 939 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 938, + "endLine": 939, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 923 + "startLine": 924 } }, { @@ -1404,15 +1404,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 912, - "endLine": 921 + "startLine": 913, + "endLine": 922 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 921, + "endLine": 922, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 912 + "startLine": 913 } }, { diff --git a/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module_chdir/expected.json b/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module_chdir/expected.json index 6ed6c4b6d82..58272378b92 100644 --- a/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module_chdir/expected.json +++ b/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module_chdir/expected.json @@ -103,15 +103,15 @@ { "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 923, - "endLine": 938 + "startLine": 924, + "endLine": 939 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 938, + "endLine": 939, "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 923 + "startLine": 924 } }, { @@ -139,15 +139,15 @@ { "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 912, - "endLine": 921 + "startLine": 913, + "endLine": 922 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 921, + "endLine": 922, "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 912 + "startLine": 913 } }, { @@ -291,15 +291,15 @@ { "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 923, - "endLine": 938 + "startLine": 924, + "endLine": 939 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 938, + "endLine": 939, "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 923 + "startLine": 924 } }, { @@ -327,15 +327,15 @@ { "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 912, - "endLine": 921 + "startLine": 913, + "endLine": 922 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 921, + "endLine": 922, "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 912 + "startLine": 913 } }, { @@ -544,15 +544,15 @@ { "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 923, - "endLine": 938 + "startLine": 924, + "endLine": 939 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 938, + "endLine": 939, "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 923 + "startLine": 924 } }, { @@ -580,15 +580,15 @@ { "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 912, - "endLine": 921 + "startLine": 913, + "endLine": 922 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 921, + "endLine": 922, "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 912 + "startLine": 913 } }, { @@ -927,15 +927,15 @@ { "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 923, - "endLine": 938 + "startLine": 924, + "endLine": 939 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 938, + "endLine": 939, "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 923 + "startLine": 924 } }, { @@ -963,15 +963,15 @@ { "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 912, - "endLine": 921 + "startLine": 913, + "endLine": 922 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 921, + "endLine": 922, "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 912 + "startLine": 913 } }, { @@ -1115,15 +1115,15 @@ { "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 923, - "endLine": 938 + "startLine": 924, + "endLine": 939 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 938, + "endLine": 939, "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 923 + "startLine": 924 } }, { @@ -1151,15 +1151,15 @@ { "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 912, - "endLine": 921 + "startLine": 913, + "endLine": 922 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 921, + "endLine": 922, "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 912 + "startLine": 913 } }, { @@ -1368,15 +1368,15 @@ { "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 923, - "endLine": 938 + "startLine": 924, + "endLine": 939 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 938, + "endLine": 939, "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 923 + "startLine": 924 } }, { @@ -1404,15 +1404,15 @@ { "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 912, - "endLine": 921 + "startLine": 913, + "endLine": 922 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 921, + "endLine": 922, "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 912 + "startLine": 913 } }, { diff --git a/internal/providers/terraform/testdata/hcl_provider_test/renders_module_resources/expected.json b/internal/providers/terraform/testdata/hcl_provider_test/renders_module_resources/expected.json index 086e6a226f5..4c461920385 100644 --- a/internal/providers/terraform/testdata/hcl_provider_test/renders_module_resources/expected.json +++ b/internal/providers/terraform/testdata/hcl_provider_test/renders_module_resources/expected.json @@ -22,15 +22,21 @@ "attributesWithUnknownKeys": [ { "attribute": "customer_gateway_id", - "missingVariables": ["module.gateway.aws_customer_gateway_id"] + "missingVariables": [ + "module.gateway.aws_customer_gateway_id" + ] }, { "attribute": "transit_gateway_id", - "missingVariables": ["module.gateway.aws_ec2_transit_gateway_id"] + "missingVariables": [ + "module.gateway.aws_ec2_transit_gateway_id" + ] }, { "attribute": "type", - "missingVariables": ["module.gateway.aws_customer_gateway_type"] + "missingVariables": [ + "module.gateway.aws_customer_gateway_type" + ] } ], "calls": [ @@ -143,15 +149,21 @@ "attributesWithUnknownKeys": [ { "attribute": "customer_gateway_id", - "missingVariables": ["module.gateway.aws_customer_gateway_id"] + "missingVariables": [ + "module.gateway.aws_customer_gateway_id" + ] }, { "attribute": "transit_gateway_id", - "missingVariables": ["module.gateway.aws_ec2_transit_gateway_id"] + "missingVariables": [ + "module.gateway.aws_ec2_transit_gateway_id" + ] }, { "attribute": "type", - "missingVariables": ["module.gateway.aws_customer_gateway_type"] + "missingVariables": [ + "module.gateway.aws_customer_gateway_type" + ] } ], "calls": [ From 55e6b075279425e41ecec944680e65c6bd2aaef3 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Fri, 11 Oct 2024 14:54:24 +0100 Subject: [PATCH 17/81] fix: index panic in Terragrunt deps: --- .../terraform/terragrunt_hcl_provider.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/internal/providers/terraform/terragrunt_hcl_provider.go b/internal/providers/terraform/terragrunt_hcl_provider.go index 49ad818cd26..f3f5ebfc101 100644 --- a/internal/providers/terraform/terragrunt_hcl_provider.go +++ b/internal/providers/terraform/terragrunt_hcl_provider.go @@ -955,17 +955,24 @@ func mergeListWithDependencyMap(valueMap map[string]cty.Value, pieces []string, if v, ok := valueMap[key]; ok && isList(v) { existing := v.AsValueSlice() - vals := make([]cty.Value, index+1) - for i, value := range existing { - if i != index && value.IsKnown() { - vals[i] = value + + s := index + 1 + if len(existing) > s { + s = len(existing) + } + vals := make([]cty.Value, s) + + for i := 0; i < len(existing); i++ { + existingVal := existing[i] + if i != index && existingVal.IsKnown() { + vals[i] = existingVal continue } // if we are at the index and the value is known, and it is an object we need to merge the object // with mock values for the rest of the pieces. - if i == index && value.IsKnown() && value.CanIterateElements() && !isList(value) { - vals[i] = cty.ObjectVal(mergeObjectWithDependencyMap(value.AsValueMap(), pieces[1:])) + if i == index && existingVal.IsKnown() && existingVal.CanIterateElements() && !isList(existingVal) { + vals[i] = cty.ObjectVal(mergeObjectWithDependencyMap(existingVal.AsValueMap(), pieces[1:])) continue } From 88c030910b9773033e8f7d6e6d4c0dddbe4d0aa9 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Tue, 15 Oct 2024 14:14:12 +0100 Subject: [PATCH 18/81] fix: additional cases for Terragrunt output mocks --- .../terraform/terragrunt_hcl_provider.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/internal/providers/terraform/terragrunt_hcl_provider.go b/internal/providers/terraform/terragrunt_hcl_provider.go index f3f5ebfc101..3a8245dff1b 100644 --- a/internal/providers/terraform/terragrunt_hcl_provider.go +++ b/internal/providers/terraform/terragrunt_hcl_provider.go @@ -783,7 +783,18 @@ var ( // - tomap(dependency.foo.bar)["baz"] // - tolist(dependency.foo.bar["baz")[0] // - tomap(dependency.foo.bar[0])["baz"] - depRegexp = regexp.MustCompile(`(?:\w+\()?dependency\.[\w\-.\[\]"]+(?:\)[\w\-.\[\]"]+)?(?:\))?`) + // - values(dependency.foo.bar).* + // - values(dependency.foo.bar).*.baz + // - values(dependency.foo.bar).baz[0] + // - values(dependency.foo.bar).baz["qux"] + // - values(dependency.foo.bar).*[0] + // - values(dependency.foo.bar).*[0]["qux"] + // - values(dependency.foo.bar).*.baz[0] + // - values(dependency.foo.bar).*.baz["qux"] + // + // The regex will also match any trailing braces so we have to strip them later on. + // There's no easy way to avoid this since golang regex doesn't support lookbehinds. + depRegexp = regexp.MustCompile(`(?:\w+\()?dependency\.[\w\-.\[\]"]+(?:\)[\w\*\-.\[\]"]+)?(?:\))?`) indexRegexp = regexp.MustCompile(`(\w+)\[(\d+)]`) mapRegexp = regexp.MustCompile(`\["([\w\d]+)"]`) ) @@ -833,7 +844,7 @@ func (p *TerragruntHCLProvider) fetchDependencyOutputs(opts *tgoptions.Terragrun valueMap := moduleOutputs.AsValueMap() for _, match := range matches { - stripped := stripFunctionCalls(match) + stripped := stripTrailingBraces(stripFunctionCalls(match)) pieces := strings.Split(stripped, ".") valueMap = mergeObjectWithDependencyMap(valueMap, pieces[1:]) } @@ -848,6 +859,10 @@ func stripFunctionCalls(input string) string { return stripped } +func stripTrailingBraces(input string) string { + return strings.TrimSuffix(input, ")") +} + func mergeObjectWithDependencyMap(valueMap map[string]cty.Value, pieces []string) map[string]cty.Value { if valueMap == nil { valueMap = make(map[string]cty.Value) From 9e7f3836a130a8efbad2024d9d886d38a745e5b3 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Tue, 15 Oct 2024 14:29:29 +0100 Subject: [PATCH 19/81] chore: actions use setup_terraform --- .github/workflows/go.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 87534ab2a09..31e5f9a339e 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -18,6 +18,9 @@ jobs: with: go-version-file: go.mod + - name: Install Terraform + uses: hashicorp/setup-terraform@v3 + - name: Check Terraform format run: make tf_fmt_check @@ -76,6 +79,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Install Terraform + uses: hashicorp/setup-terraform@v3 - name: Set up Go 1.x uses: actions/setup-go@v5 with: @@ -93,6 +98,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Install Terraform + uses: hashicorp/setup-terraform@v3 - name: Set up Go 1.x uses: actions/setup-go@v5 with: @@ -110,6 +117,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Install Terraform + uses: hashicorp/setup-terraform@v3 - name: Set up Go 1.x uses: actions/setup-go@v5 with: From 4bd5b588ef339a711cf3ff1ce7fefea650f42c2d Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Tue, 15 Oct 2024 15:58:12 +0100 Subject: [PATCH 20/81] chore: support jq 1.7 --- .github/workflows/go.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 31e5f9a339e..cbcaed45361 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -59,7 +59,7 @@ jobs: run: | make run --silent ARGS="breakdown --path examples/terraform --usage-file=examples/terraform/infracost-usage.yml --format json" > infracost-output.json [ -s infracost-output.json ] || (echo "file is empty" && exit 1) - jq '' infracost-output.json + jq '.' infracost-output.json env: INFRACOST_API_KEY: "00000000000000000000000000000000" INFRACOST_LOG_LEVEL: info @@ -68,7 +68,7 @@ jobs: run: | make run --silent ARGS="breakdown --config-file=infracost-example.yml" [ -s infracost-output.json ] || (echo "file is empty" && exit 1) - jq '' infracost-output.json + jq '.' infracost-output.json env: INFRACOST_API_KEY: "00000000000000000000000000000000" INFRACOST_LOG_LEVEL: info From 59b43d574f332dbaaf78939b0a41af0295b8345b Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Wed, 16 Oct 2024 11:24:19 +0100 Subject: [PATCH 21/81] enhance: add better mocking for conversion functions --- internal/hcl/evaluator.go | 12 +-- internal/hcl/funcs/conversion.go | 127 +++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 6 deletions(-) create mode 100644 internal/hcl/funcs/conversion.go diff --git a/internal/hcl/evaluator.go b/internal/hcl/evaluator.go index 29d014d3458..dd5bc452d74 100644 --- a/internal/hcl/evaluator.go +++ b/internal/hcl/evaluator.go @@ -1242,12 +1242,12 @@ func ExpFunctions(baseDir string, logger zerolog.Logger) map[string]function.Fun "timestamp": funcs.MockTimestampFunc, // custom. We want to return a deterministic value each time "timeadd": stdlib.TimeAddFunc, "title": stdlib.TitleFunc, - "tostring": componentsFuncs.MakeToFunc(cty.String), - "tonumber": componentsFuncs.MakeToFunc(cty.Number), - "tobool": componentsFuncs.MakeToFunc(cty.Bool), - "toset": componentsFuncs.MakeToFunc(cty.Set(cty.DynamicPseudoType)), - "tolist": componentsFuncs.MakeToFunc(cty.List(cty.DynamicPseudoType)), - "tomap": componentsFuncs.MakeToFunc(cty.Map(cty.DynamicPseudoType)), + "tostring": funcs.MakeToFunc(cty.String), + "tonumber": funcs.MakeToFunc(cty.Number), + "tobool": funcs.MakeToFunc(cty.Bool), + "toset": funcs.MakeToFunc(cty.Set(cty.DynamicPseudoType)), + "tolist": funcs.MakeToFunc(cty.List(cty.DynamicPseudoType)), + "tomap": funcs.MakeToFunc(cty.Map(cty.DynamicPseudoType)), "transpose": componentsFuncs.TransposeFunc, "trim": stdlib.TrimFunc, "trimprefix": stdlib.TrimPrefixFunc, diff --git a/internal/hcl/funcs/conversion.go b/internal/hcl/funcs/conversion.go new file mode 100644 index 00000000000..cb12ceebf64 --- /dev/null +++ b/internal/hcl/funcs/conversion.go @@ -0,0 +1,127 @@ +package funcs + +import ( + "fmt" + "strconv" + "strings" + + "github.com/infracost/infracost/internal/hcl/mock" + "github.com/turbot/terraform-components/lang/marks" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/convert" + "github.com/zclconf/go-cty/cty/function" +) + +// Adapted from https://github.com/zclconf/go-cty/blob/ea922e7a95ba2be57897697117f318670e066d22/cty/function/stdlib/conversion.go +// to handle Infracost mock values. +// +// MakeToFunc constructs a "to..." function, like "tostring", which converts +// its argument to a specific type or type kind. +// +// The given type wantTy can be any type constraint that cty's "convert" package +// would accept. In particular, this means that you can pass +// cty.List(cty.DynamicPseudoType) to mean "list of any single type", which +// will then cause cty to attempt to unify all of the element types when given +// a tuple. +func MakeToFunc(wantTy cty.Type) function.Function { + return function.New(&function.Spec{ + Description: fmt.Sprintf("Converts the given value to %s, or raises an error if that conversion is impossible.", wantTy.FriendlyName()), + Params: []function.Parameter{ + { + Name: "v", + // We use DynamicPseudoType rather than wantTy here so that + // all values will pass through the function API verbatim and + // we can handle the conversion logic within the Type and + // Impl functions. This allows us to customize the error + // messages to be more appropriate for an explicit type + // conversion, whereas the cty function system produces + // messages aimed at _implicit_ type conversions. + Type: cty.DynamicPseudoType, + AllowNull: true, + AllowMarked: true, + AllowDynamicType: true, + }, + }, + Type: func(args []cty.Value) (cty.Type, error) { + gotTy := args[0].Type() + if gotTy.Equals(wantTy) { + return wantTy, nil + } + conv := convert.GetConversionUnsafe(args[0].Type(), wantTy) + if conv == nil { + // We'll use some specialized errors for some trickier cases, + // but most we can handle in a simple way. + switch { + case gotTy.IsTupleType() && wantTy.IsTupleType(): + return cty.NilType, function.NewArgErrorf(0, "incompatible tuple type for conversion: %s", convert.MismatchMessage(gotTy, wantTy)) + case gotTy.IsObjectType() && wantTy.IsObjectType(): + return cty.NilType, function.NewArgErrorf(0, "incompatible object type for conversion: %s", convert.MismatchMessage(gotTy, wantTy)) + case gotTy == cty.String && wantTy.IsListType() || wantTy.IsTupleType() || wantTy.IsSetType() || wantTy.IsObjectType() || wantTy.IsMapType(): + val, _ := args[0].UnmarkDeep() + strVal := val.AsString() + if strings.HasSuffix(strVal, "-mock") || strings.Contains(strVal, mock.Identifier) { + return wantTy, nil + } + return cty.NilType, function.NewArgErrorf(0, "cannot convert %s to %s", gotTy.FriendlyName(), wantTy.FriendlyNameForConstraint()) + default: + return cty.NilType, function.NewArgErrorf(0, "cannot convert %s to %s", gotTy.FriendlyName(), wantTy.FriendlyNameForConstraint()) + } + } + // If a conversion is available then everything is fine. + return wantTy, nil + }, + Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { + // We didn't set "AllowUnknown" on our argument, so it is guaranteed + // to be known here but may still be null. + ret, err := convert.Convert(args[0], retType) + if err != nil { + // Because we used GetConversionUnsafe above, conversion can + // still potentially fail in here. For example, if the user + // asks to convert the string "a" to bool then we'll + // optimistically permit it during type checking but fail here + // once we note that the value isn't either "true" or "false". + val, _ := args[0].UnmarkDeep() + gotTy := val.Type() + switch { + case marks.Contains(args[0], marks.Sensitive): + return cty.NilVal, function.NewArgErrorf(0, "cannot convert sensitive type %s to %s", gotTy.FriendlyName(), wantTy.FriendlyNameForConstraint()) + case gotTy == cty.String && wantTy == cty.Bool: + what := "string" + if !val.IsNull() { + what = strconv.Quote(val.AsString()) + } + return cty.NilVal, function.NewArgErrorf(0, `cannot convert %s to bool; only the strings "true" or "false" are allowed`, what) + case gotTy == cty.String && wantTy == cty.Number: + what := "string" + if !val.IsNull() { + what = strconv.Quote(val.AsString()) + } + return cty.NilVal, function.NewArgErrorf(0, `cannot convert %s to number; given string must be a decimal representation of a number`, what) + case gotTy == cty.String && wantTy.IsListType() || wantTy.IsTupleType() || wantTy.IsSetType() || wantTy.IsObjectType() || wantTy.IsMapType(): + strVal := val.AsString() + if strings.HasSuffix(strVal, "-mock") || strings.Contains(strVal, mock.Identifier) { + switch { + case wantTy.IsListType(): + return cty.ListVal([]cty.Value{val}), nil + case wantTy.IsTupleType(): + return cty.TupleVal([]cty.Value{val}), nil + case wantTy.IsSetType(): + return cty.SetVal([]cty.Value{val}), nil + case wantTy.IsObjectType(): + return cty.ObjectVal(map[string]cty.Value{}), nil + case wantTy.IsMapType(): + return cty.MapVal(map[string]cty.Value{}), nil + default: + return cty.NilVal, function.NewArgErrorf(0, "cannot convert %s to %s", gotTy.FriendlyName(), wantTy.FriendlyNameForConstraint()) + } + } + + return cty.NilVal, function.NewArgErrorf(0, "cannot convert %s to %s", gotTy.FriendlyName(), wantTy.FriendlyNameForConstraint()) + default: + return cty.NilVal, function.NewArgErrorf(0, "cannot convert %s to %s", gotTy.FriendlyName(), wantTy.FriendlyNameForConstraint()) + } + } + return ret, nil + }, + }) +} From 38e888cfc56a42fcc2603ea4eee65350482b8202 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Wed, 16 Oct 2024 08:13:38 +0100 Subject: [PATCH 22/81] fix(aws): don't show unknown volume types as gp2 --- .../breakdown_no_prices_warnings.golden | 2 +- internal/resources/aws/ebs_volume.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/infracost/testdata/breakdown_no_prices_warnings/breakdown_no_prices_warnings.golden b/cmd/infracost/testdata/breakdown_no_prices_warnings/breakdown_no_prices_warnings.golden index 6bb14d2294c..469b0ced31c 100644 --- a/cmd/infracost/testdata/breakdown_no_prices_warnings/breakdown_no_prices_warnings.golden +++ b/cmd/infracost/testdata/breakdown_no_prices_warnings/breakdown_no_prices_warnings.golden @@ -15,7 +15,7 @@ Project: main ├─ root_block_device │ └─ Storage (general purpose SSD, gp2) 50 GB $5.00 └─ ebs_block_device[0] - └─ Storage (general purpose SSD, gp2) 1,000 GB not found + └─ Storage (unknown) 1,000 GB not found aws_instance.instance_invalid ├─ Instance usage (Linux/UNIX, on-demand, invalid_instance_type) 730 hours not found diff --git a/internal/resources/aws/ebs_volume.go b/internal/resources/aws/ebs_volume.go index 797882622a2..f2a97eed9b5 100644 --- a/internal/resources/aws/ebs_volume.go +++ b/internal/resources/aws/ebs_volume.go @@ -95,8 +95,10 @@ func (a *EBSVolume) storageCostComponent() *schema.CostComponent { name = "Storage (cold HDD, sc1)" case "gp3": name = "Storage (general purpose SSD, gp3)" - default: + case "gp2": name = "Storage (general purpose SSD, gp2)" + default: + name = "Storage (unknown)" } return &schema.CostComponent{ From 6137111a3bb899cbcb02786e52a0d0fab6a3f6eb Mon Sep 17 00:00:00 2001 From: James Nowell Date: Fri, 18 Oct 2024 13:01:42 +0100 Subject: [PATCH 23/81] fix(aws): AWS VPC Endpoint pricing tiers are wrong #3228 (#3229) --- .../vpc_endpoint_test/vpc_endpoint_test.golden | 10 +++++----- .../vpc_endpoint_test/vpc_endpoint_test.usage.yml | 2 +- internal/resources/aws/vpc_endpoint.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/providers/terraform/aws/testdata/vpc_endpoint_test/vpc_endpoint_test.golden b/internal/providers/terraform/aws/testdata/vpc_endpoint_test/vpc_endpoint_test.golden index 502341065ab..a99536b64fc 100644 --- a/internal/providers/terraform/aws/testdata/vpc_endpoint_test/vpc_endpoint_test.golden +++ b/internal/providers/terraform/aws/testdata/vpc_endpoint_test/vpc_endpoint_test.golden @@ -2,9 +2,9 @@ Name Monthly Qty Unit Monthly Cost aws_vpc_endpoint.interface_withBigUsage - ├─ Data processed (first 1PB) 1,000 GB $10.00 * - ├─ Data processed (next 4PB) 4,000 GB $24.00 * - ├─ Data processed (over 5PB) 2,000 GB $8.00 * + ├─ Data processed (first 1PB) 1,000,000 GB $10,000.00 * + ├─ Data processed (next 4PB) 4,000,000 GB $24,000.00 * + ├─ Data processed (over 5PB) 2,000,000 GB $8,000.00 * └─ Endpoint (Interface) 730 hours $7.30 aws_vpc_endpoint.interface_withUsage @@ -27,7 +27,7 @@ ├─ Data processed (first 1PB) Monthly cost depends on usage: $0.01 per GB └─ Endpoint (Interface) 730 hours $7.30 - OVERALL TOTAL $110.40 + OVERALL TOTAL $42,068.40 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -39,5 +39,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $58 ┃ $52 ┃ $110 ┃ +┃ main ┃ $58 ┃ $42,010 ┃ $42,068 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/aws/testdata/vpc_endpoint_test/vpc_endpoint_test.usage.yml b/internal/providers/terraform/aws/testdata/vpc_endpoint_test/vpc_endpoint_test.usage.yml index 1254813f618..b0903e75e78 100644 --- a/internal/providers/terraform/aws/testdata/vpc_endpoint_test/vpc_endpoint_test.usage.yml +++ b/internal/providers/terraform/aws/testdata/vpc_endpoint_test/vpc_endpoint_test.usage.yml @@ -4,4 +4,4 @@ resource_usage: monthly_data_processed_gb: 1000 aws_vpc_endpoint.interface_withBigUsage: - monthly_data_processed_gb: 7000 + monthly_data_processed_gb: 7000000 diff --git a/internal/resources/aws/vpc_endpoint.go b/internal/resources/aws/vpc_endpoint.go index 17d70cb6ba6..a0ee871a526 100644 --- a/internal/resources/aws/vpc_endpoint.go +++ b/internal/resources/aws/vpc_endpoint.go @@ -67,7 +67,7 @@ func (r *VPCEndpoint) BuildResource() *schema.Resource { endpointHours = "VpcEndpoint-Hours" endpointBytes = "VpcEndpoint-Bytes" if dataProcessedGB != nil { - gbLimits := []int{1000, 4000} + gbLimits := []int{1000000, 4000000} tiers := usage.CalculateTierBuckets(*dataProcessedGB, gbLimits) if tiers[0].GreaterThan(decimal.NewFromInt(0)) { From e7d75f1a1a05289f959aeac04a8b355c41ce2faa Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Sat, 19 Oct 2024 09:15:05 +0100 Subject: [PATCH 24/81] fix: add mocking for `data.aws_subnets` (#3230) Previously this data attribute got mocked to a string value. That meant if users were doing anything like `length(data.aws_subnets.private_subnets.ids)` this could return high numbers, which was massively affecting any cost estimates where this was being used in a `count` or `for_each`. Although we can't actually know the number of subnets, we can mock it to a more sensible value. 3 seems like the most common number of subnets used. --- internal/hcl/block.go | 19 +++++++++++++++++++ internal/hcl/parser_test.go | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/internal/hcl/block.go b/internal/hcl/block.go index ee94f8ae8f3..050482119dd 100644 --- a/internal/hcl/block.go +++ b/internal/hcl/block.go @@ -19,6 +19,7 @@ import ( "github.com/zclconf/go-cty/cty/gocty" "github.com/infracost/infracost/internal/hcl/funcs" + "github.com/infracost/infracost/internal/hcl/mock" "github.com/infracost/infracost/internal/hcl/modules" ) @@ -1381,6 +1382,7 @@ var ( "data.google_compute_zones": googleComputeZonesValues, "data.aws_region": awsCurrentRegion, "data.aws_default_tags": awsDefaultTagValues, + "data.aws_subnets": awsSubnetsValues, "resource.random_shuffle": randomShuffleValues, "resource.time_static": timeStaticValues, "resource.aws_launch_template": launchTemplateValues, @@ -1464,6 +1466,23 @@ func awsDefaultTagValues(b *Block) cty.Value { }) } +// awsSubnetsValues mocks the values returned from data.aws_subnets. This data +// source returns a list of subnet IDs. +// https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets +// +// We return a list of 3 fake subnet IDs. Although the actual number of subnets +// returned is based on the region and availability zones, we return a fixed +// "sensible" number. +func awsSubnetsValues(b *Block) cty.Value { + return cty.ObjectVal(map[string]cty.Value{ + "ids": cty.ListVal([]cty.Value{ + cty.StringVal(fmt.Sprintf("subnet-1-%s", mock.Identifier)), + cty.StringVal(fmt.Sprintf("subnet-2-%s", mock.Identifier)), + cty.StringVal(fmt.Sprintf("subnet-3-%s", mock.Identifier)), + }), + }) +} + // randomShuffleValues mocks the values returned from resource.random_shuffle // https://github.com/hashicorp/terraform-provider-random/blob/main/docs/resources/shuffle.md. // This resource uses the result_count attribute to return a slice of the input diff --git a/internal/hcl/parser_test.go b/internal/hcl/parser_test.go index 4dd029b9f72..edf9a110826 100644 --- a/internal/hcl/parser_test.go +++ b/internal/hcl/parser_test.go @@ -1520,6 +1520,39 @@ resource "random_shuffle" "bad" { ) } +func Test_MocksForAWSSubnets(t *testing.T) { + path := createTestFile("test.tf", ` +provider "aws" { + region = "us-east-1" +} + +data "aws_subnets" "example" { + filter { + name = "vpc-id" + values = ["vpc-12345678"] + } +}`) + + logger := newDiscardLogger() + loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + parser := NewParser( + RootPath{DetectedPath: filepath.Dir(path)}, + CreateEnvFileMatcher([]string{}, nil), + loader, + logger, + ) + module, err := parser.ParseDirectory() + require.NoError(t, err) + + blocks := module.Blocks + assertBlockEqualsJSON( + t, + `{"ids":["subnet-1-infracost-mock-44956be29f34", "subnet-2-infracost-mock-44956be29f34", "subnet-3-infracost-mock-44956be29f34"]}`, + blocks.Matching(BlockMatcher{Label: "aws_subnets.example", Type: "data"}).Values(), + "id", "arn", "self_link", "name", + ) +} + func Test_LocalsMergeWithDataTags(t *testing.T) { path := createTestFile("test.tf", ` provider "aws" { @@ -2105,7 +2138,7 @@ provider "aws" { content: ` provider "aws" { default_tags { - tags = merge(var.default_tags, { + tags = merge(var.default_tags, { "Environment" = "Test" }) } From d7a14cb35aeb070cbccea57547446ae0eafaa1f1 Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 23 Oct 2024 09:20:51 +0100 Subject: [PATCH 25/81] fix: No duplicate dependencies (#3231) - add function to unique the sorted dependencies - add tests to validate --- internal/hcl/parser.go | 21 +- internal/hcl/parser_test.go | 60 ++++- .../sns_topic_test/sns_topic_test.golden | 10 +- .../virtual_machine_test.golden | 218 +++++++++--------- .../compute_instance_test.golden | 12 +- .../container_cluster_test.golden | 34 +-- .../container_node_pool_test.golden | 20 +- 7 files changed, 211 insertions(+), 164 deletions(-) diff --git a/internal/hcl/parser.go b/internal/hcl/parser.go index c2f4c166410..2961749823f 100644 --- a/internal/hcl/parser.go +++ b/internal/hcl/parser.go @@ -14,6 +14,7 @@ import ( "github.com/rs/zerolog" "github.com/zclconf/go-cty/cty" "github.com/zclconf/go-cty/cty/gocty" + "golang.org/x/exp/maps" "github.com/infracost/infracost/internal/clierror" "github.com/infracost/infracost/internal/config" @@ -22,9 +23,7 @@ import ( "github.com/infracost/infracost/internal/logging" ) -var ( - defaultTerraformWorkspaceName = "default" -) +var defaultTerraformWorkspaceName = "default" type Option func(p *Parser) @@ -411,9 +410,21 @@ func (p *Parser) DependencyPaths() []string { } else { sortedCalls = append(sortedCalls, p.RelativePath()+"/**") } - sort.Strings(sortedCalls) + return p.uniqueDependencyPaths(sortedCalls) +} + +func (p *Parser) uniqueDependencyPaths(paths []string) []string { + seen := make(map[string]struct{}) - return sortedCalls + for _, path := range paths { + if _, ok := seen[path]; ok { + continue + } + seen[path] = struct{}{} + } + deps := maps.Keys(seen) + sort.Strings(deps) + return deps } // ParseDirectory parses all the terraform files in the detectedProjectPath into Blocks and then passes them to an Evaluator diff --git a/internal/hcl/parser_test.go b/internal/hcl/parser_test.go index edf9a110826..699a7c1f910 100644 --- a/internal/hcl/parser_test.go +++ b/internal/hcl/parser_test.go @@ -194,7 +194,6 @@ output "loadbalancer" { } require.Len(t, keys, 1) - } func Test_UnsupportedAttributesList(t *testing.T) { @@ -542,7 +541,6 @@ output "val" { } func Test_Modules(t *testing.T) { - path := createTestFileWithModule(` module "my-mod" { source = "../module" @@ -604,11 +602,9 @@ output "mod_result" { require.NotNil(t, childValAttr) require.Equal(t, cty.String, childValAttr.Value().Type()) assert.Equal(t, "ok", childValAttr.Value().AsString()) - } func Test_NestedParentModule(t *testing.T) { - path := createTestFileWithModule(` module "my-mod" { source = "../." @@ -917,7 +913,6 @@ resource "test_resource_two" "test" { } func Test_ModuleForEaches(t *testing.T) { - path := createTestFileWithModule(` locals { az = { @@ -1227,7 +1222,6 @@ resource "dynamic" "resource" { ]`, values, ) - } func Test_ForEachReferencesAnotherForEachDependentAttribute(t *testing.T) { @@ -2107,11 +2101,9 @@ terraform { assert.Equal(t, "~> 5.52.0", module.ProviderConstraints.AWS.String()) assert.Equal(t, ">= 5.0.0,< 5.0.4", module.ProviderConstraints.Google.String()) - } func Test_UnknownKeyDetection(t *testing.T) { - tests := []struct { name string content string @@ -2227,9 +2219,7 @@ resource "aws_instance" "blah" { } for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - variations := []struct { name string parserOpts []Option @@ -2245,9 +2235,7 @@ resource "aws_instance" "blah" { } for _, variation := range variations { - t.Run(variation.name, func(t *testing.T) { - path := createTestFile("test.tf", tt.content) logger := newDiscardLogger() @@ -2286,3 +2274,51 @@ resource "aws_instance" "blah" { }) } } + +func Test_uniqueDependencyPaths(t *testing.T) { + tests := []struct { + name string + paths []string + expected []string + }{ + { + name: "no duplicates", + paths: []string{ + "module.foo", + "module.bar", + "module.baz", + }, + expected: []string{ + "\"**\"", + "module.bar/**", + "module.baz/**", + "module.foo/**", + }, + }, + { + name: "duplicates", + paths: []string{ + "module.foo", + "module.bar", + "module.foo", + "module.baz", + "module.bar", + }, + expected: []string{ + "\"**\"", + "module.bar/**", + "module.baz/**", + "module.foo/**", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := Parser{ + moduleCalls: tt.paths, + } + result := p.DependencyPaths() + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/internal/providers/terraform/aws/testdata/sns_topic_test/sns_topic_test.golden b/internal/providers/terraform/aws/testdata/sns_topic_test/sns_topic_test.golden index 9bb3e718973..0d589e9a713 100644 --- a/internal/providers/terraform/aws/testdata/sns_topic_test/sns_topic_test.golden +++ b/internal/providers/terraform/aws/testdata/sns_topic_test/sns_topic_test.golden @@ -7,7 +7,7 @@ ├─ Kinesis Firehose notifications 4 1M notifications $0.76 * ├─ Mobile Push notifications 5 1M notifications $2.50 * ├─ MacOS notifications 6 1M notifications $3.00 * - └─ SMS notifications (over 100) 9 100 notifications $6.75 * + └─ SMS notifications (over 100) 9 100 notifications not found * aws_sns_topic.sns_topic_customSmsPrice └─ SMS notifications (over 100) 9 100 notifications $8.88 * @@ -52,7 +52,7 @@ ├─ Kinesis Firehose notifications Monthly cost depends on usage: $0.19 per 1M notifications ├─ Mobile Push notifications Monthly cost depends on usage: $0.50 per 1M notifications ├─ MacOS notifications Monthly cost depends on usage: $0.50 per 1M notifications - └─ SMS notifications (over 100) Monthly cost depends on usage: $0.75 per 100 notifications + └─ SMS notifications (over 100) not found aws_sns_topic.sns_topic_another_region ├─ API requests (over 1M) Monthly cost depends on usage: $0.50 per 1M requests @@ -61,9 +61,9 @@ ├─ Kinesis Firehose notifications Monthly cost depends on usage: $0.19 per 1M notifications ├─ Mobile Push notifications Monthly cost depends on usage: $0.50 per 1M notifications ├─ MacOS notifications Monthly cost depends on usage: $0.50 per 1M notifications - └─ SMS notifications (over 100) Monthly cost depends on usage: $0.75 per 100 notifications + └─ SMS notifications (over 100) not found - OVERALL TOTAL $34.13 + OVERALL TOTAL $27.38 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -75,5 +75,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $0.00 ┃ $34 ┃ $34 ┃ +┃ main ┃ $0.00 ┃ $27 ┃ $27 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/virtual_machine_test/virtual_machine_test.golden b/internal/providers/terraform/azure/testdata/virtual_machine_test/virtual_machine_test.golden index 952ab6e557b..2518eb1d66b 100644 --- a/internal/providers/terraform/azure/testdata/virtual_machine_test/virtual_machine_test.golden +++ b/internal/providers/terraform/azure/testdata/virtual_machine_test/virtual_machine_test.golden @@ -141,6 +141,13 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations + azurerm_virtual_machine.windows_vms["Standard_E96ds_v6"] + ├─ Instance usage (Windows, pay as you go, Standard_E96ds_v6) 730 hours $8,953.45 + ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU + └─ storage_os_disk + ├─ Storage (S4, LRS) 1 months $1.54 + └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations + azurerm_virtual_machine.linux_vms["Standard_ND40s_v2"] ├─ Instance usage (Linux, pay as you go, Standard_ND40s_v2) 730 hours $8,935.20 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU @@ -155,6 +162,13 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations + azurerm_virtual_machine.windows_vms["Standard_E96ads_v6"] + ├─ Instance usage (Windows, pay as you go, Standard_E96ads_v6) 730 hours $8,311.78 + ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU + └─ storage_os_disk + ├─ Storage (S4, LRS) 1 months $1.54 + └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations + azurerm_virtual_machine.windows_vms["Standard_E96iads_v5"] ├─ Instance usage (Windows, pay as you go, Standard_E96iads_v5) 730 hours $8,273.09 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU @@ -190,13 +204,6 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_E96ads_v6"] - ├─ Instance usage (Windows, pay as you go, Standard_E96ads_v6) 730 hours $7,848.96 - ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU - └─ storage_os_disk - ├─ Storage (S4, LRS) 1 months $1.54 - └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_E96ads_v5"] ├─ Instance usage (Windows, pay as you go, Standard_E96ads_v5) 730 hours $7,813.92 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU @@ -260,6 +267,20 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations + azurerm_virtual_machine.windows_vms["Standard_E96as_v6"] + ├─ Instance usage (Windows, pay as you go, Standard_E96as_v6) 730 hours $7,400.74 + ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU + └─ storage_os_disk + ├─ Storage (S4, LRS) 1 months $1.54 + └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations + + azurerm_virtual_machine.windows_vms["Standard_D96ads_v6"] + ├─ Instance usage (Windows, pay as you go, Standard_D96ads_v6) 730 hours $7,218.24 + ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU + └─ storage_os_disk + ├─ Storage (S4, LRS) 1 months $1.54 + └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations + azurerm_virtual_machine.windows_vms["Standard_D96d_v5"] ├─ Instance usage (Windows, pay as you go, Standard_D96d_v5) 730 hours $7,183.20 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU @@ -302,13 +323,6 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_E96as_v6"] - ├─ Instance usage (Windows, pay as you go, Standard_E96as_v6) 730 hours $7,021.87 - ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU - └─ storage_os_disk - ├─ Storage (S4, LRS) 1 months $1.54 - └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_M64ds_v2"] ├─ Instance usage (Windows, pay as you go, Standard_M64ds_v2) 730 hours $7,017.49 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU @@ -323,13 +337,6 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_D96ads_v6"] - ├─ Instance usage (Windows, pay as you go, Standard_D96ads_v6) 730 hours $6,860.54 - ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU - └─ storage_os_disk - ├─ Storage (S4, LRS) 1 months $1.54 - └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_D96ads_v5"] ├─ Instance usage (Windows, pay as you go, Standard_D96ads_v5) 730 hours $6,832.80 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU @@ -407,22 +414,29 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_D96ls_v6"] - ├─ Instance usage (Windows, pay as you go, Standard_D96ls_v6) 730 hours $6,351.00 + azurerm_virtual_machine.windows_vms["Standard_D96alds_v6"] + ├─ Instance usage (Windows, pay as you go, Standard_D96alds_v6) 730 hours $6,559.78 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.linux_vms["Standard_HX176rs"] - ├─ Instance usage (Linux, pay as you go, Standard_HX176rs) 730 hours $6,307.20 + azurerm_virtual_machine.windows_vms["Standard_D96as_v6"] + ├─ Instance usage (Windows, pay as you go, Standard_D96as_v6) 730 hours $6,405.02 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_D96alds_v6"] - ├─ Instance usage (Windows, pay as you go, Standard_D96alds_v6) 730 hours $6,258.29 + azurerm_virtual_machine.windows_vms["Standard_D96ls_v6"] + ├─ Instance usage (Windows, pay as you go, Standard_D96ls_v6) 730 hours $6,351.00 + ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU + └─ storage_os_disk + ├─ Storage (S4, LRS) 1 months $1.54 + └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations + + azurerm_virtual_machine.linux_vms["Standard_HX176rs"] + ├─ Instance usage (Linux, pay as you go, Standard_HX176rs) 730 hours $6,307.20 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 @@ -463,8 +477,8 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_D96as_v6"] - ├─ Instance usage (Windows, pay as you go, Standard_D96as_v6) 730 hours $6,118.13 + azurerm_virtual_machine.windows_vms["Standard_D96als_v6"] + ├─ Instance usage (Windows, pay as you go, Standard_D96als_v6) 730 hours $6,040.75 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 @@ -477,8 +491,8 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_D96als_v6"] - ├─ Instance usage (Windows, pay as you go, Standard_D96als_v6) 730 hours $5,781.60 + azurerm_virtual_machine.linux_vms["Standard_E96ds_v6"] + ├─ Instance usage (Linux, pay as you go, Standard_E96ds_v6) 730 hours $5,729.77 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 @@ -498,6 +512,13 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations + azurerm_virtual_machine.linux_vms["Standard_E96ads_v6"] + ├─ Instance usage (Linux, pay as you go, Standard_E96ads_v6) 730 hours $5,088.10 + ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU + └─ storage_os_disk + ├─ Storage (S4, LRS) 1 months $1.54 + └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations + azurerm_virtual_machine.linux_vms["Standard_E96iads_v5"] ├─ Instance usage (Linux, pay as you go, Standard_E96iads_v5) 730 hours $5,049.41 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU @@ -568,13 +589,6 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.linux_vms["Standard_E96ads_v6"] - ├─ Instance usage (Linux, pay as you go, Standard_E96ads_v6) 730 hours $4,625.28 - ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU - └─ storage_os_disk - ├─ Storage (S4, LRS) 1 months $1.54 - └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.linux_vms["Standard_E96ads_v5"] ├─ Instance usage (Linux, pay as you go, Standard_E96ads_v5) 730 hours $4,590.24 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU @@ -638,50 +652,50 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.linux_vms["Standard_D96d_v5"] - ├─ Instance usage (Linux, pay as you go, Standard_D96d_v5) 730 hours $3,959.52 + azurerm_virtual_machine.linux_vms["Standard_E96as_v6"] + ├─ Instance usage (Linux, pay as you go, Standard_E96as_v6) 730 hours $4,177.06 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.linux_vms["Standard_D96ds_v5"] - ├─ Instance usage (Linux, pay as you go, Standard_D96ds_v5) 730 hours $3,959.52 + azurerm_virtual_machine.linux_vms["Standard_D96ads_v6"] + ├─ Instance usage (Linux, pay as you go, Standard_D96ads_v6) 730 hours $3,994.56 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.linux_vms["Standard_E96as_v5"] - ├─ Instance usage (Linux, pay as you go, Standard_E96as_v5) 730 hours $3,959.52 + azurerm_virtual_machine.linux_vms["Standard_D96d_v5"] + ├─ Instance usage (Linux, pay as you go, Standard_D96d_v5) 730 hours $3,959.52 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.linux_vms["Standard_EC96as_cc_v5"] - ├─ Instance usage (Linux, pay as you go, Standard_EC96as_cc_v5) 730 hours $3,959.52 + azurerm_virtual_machine.linux_vms["Standard_D96ds_v5"] + ├─ Instance usage (Linux, pay as you go, Standard_D96ds_v5) 730 hours $3,959.52 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_E96ds_v6"] - ├─ Instance usage (Windows, pay as you go, Standard_E96ds_v6) 730 hours $3,924.48 + azurerm_virtual_machine.linux_vms["Standard_E96as_v5"] + ├─ Instance usage (Linux, pay as you go, Standard_E96as_v5) 730 hours $3,959.52 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.linux_vms["Standard_D96lds_v6"] - ├─ Instance usage (Linux, pay as you go, Standard_D96lds_v6) 730 hours $3,865.35 + azurerm_virtual_machine.linux_vms["Standard_EC96as_cc_v5"] + ├─ Instance usage (Linux, pay as you go, Standard_EC96as_cc_v5) 730 hours $3,959.52 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.linux_vms["Standard_E96as_v6"] - ├─ Instance usage (Linux, pay as you go, Standard_E96as_v6) 730 hours $3,798.19 + azurerm_virtual_machine.linux_vms["Standard_D96lds_v6"] + ├─ Instance usage (Linux, pay as you go, Standard_D96lds_v6) 730 hours $3,865.35 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 @@ -701,13 +715,6 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.linux_vms["Standard_D96ads_v6"] - ├─ Instance usage (Linux, pay as you go, Standard_D96ads_v6) 730 hours $3,636.86 - ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU - └─ storage_os_disk - ├─ Storage (S4, LRS) 1 months $1.54 - └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.linux_vms["Standard_D96ads_v5"] ├─ Instance usage (Linux, pay as you go, Standard_D96ads_v5) 730 hours $3,609.12 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU @@ -771,6 +778,13 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations + azurerm_virtual_machine.linux_vms["Standard_D96alds_v6"] + ├─ Instance usage (Linux, pay as you go, Standard_D96alds_v6) 730 hours $3,336.10 + ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU + └─ storage_os_disk + ├─ Storage (S4, LRS) 1 months $1.54 + └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations + azurerm_virtual_machine.linux_vms["Standard_NV48s_v3"] ├─ Instance usage (Linux, pay as you go, Standard_NV48s_v3) 730 hours $3,328.80 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU @@ -778,15 +792,15 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.linux_vms["Standard_D96ls_v6"] - ├─ Instance usage (Linux, pay as you go, Standard_D96ls_v6) 730 hours $3,127.32 + azurerm_virtual_machine.linux_vms["Standard_D96as_v6"] + ├─ Instance usage (Linux, pay as you go, Standard_D96as_v6) 730 hours $3,181.34 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.linux_vms["Standard_D96alds_v6"] - ├─ Instance usage (Linux, pay as you go, Standard_D96alds_v6) 730 hours $3,034.61 + azurerm_virtual_machine.linux_vms["Standard_D96ls_v6"] + ├─ Instance usage (Linux, pay as you go, Standard_D96ls_v6) 730 hours $3,127.32 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 @@ -820,8 +834,8 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.linux_vms["Standard_D96as_v6"] - ├─ Instance usage (Linux, pay as you go, Standard_D96as_v6) 730 hours $2,894.45 + azurerm_virtual_machine.linux_vms["Standard_D96als_v6"] + ├─ Instance usage (Linux, pay as you go, Standard_D96als_v6) 730 hours $2,817.07 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 @@ -841,13 +855,6 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.linux_vms["Standard_D96als_v6"] - ├─ Instance usage (Linux, pay as you go, Standard_D96als_v6) 730 hours $2,557.92 - ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU - └─ storage_os_disk - ├─ Storage (S4, LRS) 1 months $1.54 - └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_NC6s_v3"] ├─ Instance usage (Windows, pay as you go, Standard_NC6s_v3) 730 hours $2,435.28 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU @@ -1037,15 +1044,15 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_F8ams_v6"] - ├─ Instance usage (Windows, pay as you go, Standard_F8ams_v6) 730 hours $811.03 + azurerm_virtual_machine.windows_vms["Standard_PB6s"] + ├─ Instance usage (Windows, pay as you go, Standard_PB6s) 730 hours $807.38 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_PB6s"] - ├─ Instance usage (Windows, pay as you go, Standard_PB6s) 730 hours $807.38 + azurerm_virtual_machine.windows_vms["Standard_F8ams_v6"] + ├─ Instance usage (Windows, pay as you go, Standard_F8ams_v6) 730 hours $791.32 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 @@ -1086,13 +1093,6 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.linux_vms["Standard_E96ds_v6"] - ├─ Instance usage (Linux, pay as you go, Standard_E96ds_v6) 730 hours $700.80 - ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU - └─ storage_os_disk - ├─ Storage (S4, LRS) 1 months $1.54 - └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_E8d_v4"] ├─ Instance usage (Windows, pay as you go, Standard_E8d_v4) 730 hours $689.12 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU @@ -1107,15 +1107,15 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_F8as_v6"] - ├─ Instance usage (Windows, pay as you go, Standard_F8as_v6) 730 hours $683.28 + azurerm_virtual_machine.windows_vms["Standard_F8alds_v6"] + ├─ Instance usage (Windows, pay as you go, Standard_F8alds_v6) 730 hours $682.55 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_F8alds_v6"] - ├─ Instance usage (Windows, pay as you go, Standard_F8alds_v6) 730 hours $682.55 + azurerm_virtual_machine.windows_vms["Standard_F8as_v6"] + ├─ Instance usage (Windows, pay as you go, Standard_F8as_v6) 730 hours $667.22 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 @@ -1163,15 +1163,15 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_F8als_v6"] - ├─ Instance usage (Windows, pay as you go, Standard_F8als_v6) 730 hours $635.10 + azurerm_virtual_machine.linux_vms["Standard_F8amds_v6"] + ├─ Instance usage (Linux, pay as you go, Standard_F8amds_v6) 730 hours $629.26 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.linux_vms["Standard_F8amds_v6"] - ├─ Instance usage (Linux, pay as you go, Standard_F8amds_v6) 730 hours $629.26 + azurerm_virtual_machine.windows_vms["Standard_F8als_v6"] + ├─ Instance usage (Windows, pay as you go, Standard_F8als_v6) 730 hours $621.96 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 @@ -1282,22 +1282,22 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.linux_vms["Standard_F8ams_v6"] - ├─ Instance usage (Linux, pay as you go, Standard_F8ams_v6) 730 hours $542.39 + azurerm_virtual_machine.windows_vms["Standard_D8pds_v5"] + ├─ Instance usage (Windows, pay as you go, Standard_D8pds_v5) 730 hours $532.90 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_D8pds_v5"] - ├─ Instance usage (Windows, pay as you go, Standard_D8pds_v5) 730 hours $532.90 + azurerm_virtual_machine.windows_vms["Standard_A8m_v2"] + ├─ Instance usage (Windows, pay as you go, Standard_A8m_v2) 730 hours $525.60 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_A8m_v2"] - ├─ Instance usage (Windows, pay as you go, Standard_A8m_v2) 730 hours $525.60 + azurerm_virtual_machine.linux_vms["Standard_F8ams_v6"] + ├─ Instance usage (Linux, pay as you go, Standard_F8ams_v6) 730 hours $522.68 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 @@ -1394,22 +1394,22 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.linux_vms["Standard_F8as_v6"] - ├─ Instance usage (Linux, pay as you go, Standard_F8as_v6) 730 hours $414.64 + azurerm_virtual_machine.linux_vms["Standard_F8alds_v6"] + ├─ Instance usage (Linux, pay as you go, Standard_F8alds_v6) 730 hours $413.91 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.linux_vms["Standard_F8alds_v6"] - ├─ Instance usage (Linux, pay as you go, Standard_F8alds_v6) 730 hours $413.91 + azurerm_virtual_machine.windows_vms["Standard_FX4mds"] + ├─ Instance usage (Windows, pay as you go, Standard_FX4mds) 730 hours $405.88 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_FX4mds"] - ├─ Instance usage (Windows, pay as you go, Standard_FX4mds) 730 hours $405.88 + azurerm_virtual_machine.linux_vms["Standard_F8as_v6"] + ├─ Instance usage (Linux, pay as you go, Standard_F8as_v6) 730 hours $398.58 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 @@ -1443,15 +1443,15 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.linux_vms["Standard_F8als_v6"] - ├─ Instance usage (Linux, pay as you go, Standard_F8als_v6) 730 hours $366.46 + azurerm_virtual_machine.windows_vms["Standard_DC4s"] + ├─ Instance usage (Windows, pay as you go, Standard_DC4s) 730 hours $355.51 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - azurerm_virtual_machine.windows_vms["Standard_DC4s"] - ├─ Instance usage (Windows, pay as you go, Standard_DC4s) 730 hours $355.51 + azurerm_virtual_machine.linux_vms["Standard_F8als_v6"] + ├─ Instance usage (Linux, pay as you go, Standard_F8als_v6) 730 hours $353.32 ├─ Ultra disk reservation (if unattached) Monthly cost depends on usage: $4.38 per vCPU └─ storage_os_disk ├─ Storage (S4, LRS) 1 months $1.54 @@ -1693,7 +1693,7 @@ ├─ Storage (S4, LRS) 1 months $1.54 └─ Disk operations Monthly cost depends on usage: $0.0005 per 10k operations - OVERALL TOTAL $1,039,464.56 + OVERALL TOTAL $1,053,518.52 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -1705,5 +1705,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $1,039,464 ┃ $0.55 ┃ $1,039,465 ┃ +┃ main ┃ $1,053,518 ┃ $0.55 ┃ $1,053,519 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/google/testdata/compute_instance_test/compute_instance_test.golden b/internal/providers/terraform/google/testdata/compute_instance_test/compute_instance_test.golden index 8189d856901..9ef4a6cef26 100644 --- a/internal/providers/terraform/google/testdata/compute_instance_test/compute_instance_test.golden +++ b/internal/providers/terraform/google/testdata/compute_instance_test/compute_instance_test.golden @@ -16,9 +16,9 @@ └─ NVIDIA L4 (on-demand) 730 hours $286.18 google_compute_instance.preemptible_gpu - ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 730 hours $138.34 + ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 730 hours $152.19 ├─ Standard provisioned storage (pd-standard) 10 GB $0.40 - └─ NVIDIA Tesla K80 (preemptible) 2,920 hours $436.54 + └─ NVIDIA Tesla K80 (preemptible) 2,920 hours $501.95 google_compute_instance.sud_20_perc_with_hours ├─ Instance usage (Linux/UNIX, on-demand, n2-standard-8) 730 hours $226.87 @@ -56,8 +56,8 @@ └─ Local SSD provisioned storage 750 GB $60.00 google_compute_instance.custom_preemptible - ├─ Custom instance CPU (Linux/UNIX, preemptible, N1 6 vCPUs) 730 hours $36.79 - ├─ Custom Instance RAM (Linux/UNIX, preemptible, N1 20 GB) 730 hours $15.71 + ├─ Custom instance CPU (Linux/UNIX, preemptible, N1 6 vCPUs) 730 hours $40.47 + ├─ Custom Instance RAM (Linux/UNIX, preemptible, N1 20 GB) 730 hours $17.29 └─ Standard provisioned storage (pd-standard) 10 GB $0.40 google_compute_instance.custom_n2d @@ -86,7 +86,7 @@ ├─ Instance usage (Linux/UNIX, on-demand, f1-micro) 100 hours $0.76 └─ Standard provisioned storage (pd-standard) 10 GB $0.40 - OVERALL TOTAL $10,271.61 + OVERALL TOTAL $10,356.12 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -99,7 +99,7 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $10,272 ┃ - ┃ $10,272 ┃ +┃ main ┃ $10,356 ┃ - ┃ $10,356 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ Logs: diff --git a/internal/providers/terraform/google/testdata/container_cluster_test/container_cluster_test.golden b/internal/providers/terraform/google/testdata/container_cluster_test/container_cluster_test.golden index f3c895ee4d9..27a21705262 100644 --- a/internal/providers/terraform/google/testdata/container_cluster_test/container_cluster_test.golden +++ b/internal/providers/terraform/google/testdata/container_cluster_test/container_cluster_test.golden @@ -10,7 +10,7 @@ │ ├─ Instance usage (Linux/UNIX, on-demand, n1-standard-16) 8,760 hours $4,660.30 │ └─ Standard provisioned storage (pd-standard) 1,200 GB $48.00 └─ node_pool[1] - ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 8,760 hours $1,660.02 + ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 8,760 hours $1,826.28 └─ Standard provisioned storage (pd-standard) 1,200 GB $48.00 google_container_cluster.with_node_pools_regional @@ -22,7 +22,7 @@ │ ├─ Instance usage (Linux/UNIX, on-demand, n1-standard-16) 4,380 hours $2,330.15 │ └─ Standard provisioned storage (pd-standard) 600 GB $24.00 └─ node_pool[1] - ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 8,760 hours $1,660.02 + ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 8,760 hours $1,826.28 └─ Standard provisioned storage (pd-standard) 1,200 GB $48.00 google_container_cluster.with_node_config @@ -42,7 +42,7 @@ │ ├─ Instance usage (Linux/UNIX, on-demand, n1-standard-16) 5,840 hours $3,106.86 │ └─ Standard provisioned storage (pd-standard) 800 GB $32.00 └─ node_pool[1] - ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 2,920 hours $553.34 + ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 2,920 hours $608.76 └─ Standard provisioned storage (pd-standard) 400 GB $16.00 google_container_cluster.with_node_pools_zonal_withUsage @@ -54,9 +54,18 @@ │ ├─ Instance usage (Linux/UNIX, on-demand, n1-standard-16) 2,920 hours $1,553.43 │ └─ Standard provisioned storage (pd-standard) 400 GB $16.00 └─ node_pool[1] - ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 2,920 hours $553.34 + ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 2,920 hours $608.76 └─ Standard provisioned storage (pd-standard) 400 GB $16.00 + google_container_cluster.with_unsupported_node_pool + ├─ Cluster management fee 730 hours $73.00 + ├─ default_pool + │ ├─ Instance usage (Linux/UNIX, on-demand, e2-medium) 6,570 hours $220.13 + │ └─ Standard provisioned storage (pd-standard) 900 GB $36.00 + └─ node_pool[1] + ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 8,760 hours $1,826.28 + └─ Standard provisioned storage (pd-standard) 1,200 GB $48.00 + google_container_cluster.with_node_pools_node_locations ├─ Cluster management fee 730 hours $73.00 ├─ default_pool @@ -66,18 +75,9 @@ │ ├─ Instance usage (Linux/UNIX, on-demand, n1-standard-16) 2,920 hours $1,553.43 │ └─ Standard provisioned storage (pd-standard) 400 GB $16.00 └─ node_pool[1] - ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 1,460 hours $276.67 + ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 1,460 hours $304.38 └─ Standard provisioned storage (pd-standard) 200 GB $8.00 - google_container_cluster.with_unsupported_node_pool - ├─ Cluster management fee 730 hours $73.00 - ├─ default_pool - │ ├─ Instance usage (Linux/UNIX, on-demand, e2-medium) 6,570 hours $220.13 - │ └─ Standard provisioned storage (pd-standard) 900 GB $36.00 - └─ node_pool[1] - ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 8,760 hours $1,660.02 - └─ Standard provisioned storage (pd-standard) 1,200 GB $48.00 - google_container_cluster.with_node_pools_zonal ├─ Cluster management fee 730 hours $73.00 ├─ default_pool @@ -87,7 +87,7 @@ │ ├─ Instance usage (Linux/UNIX, on-demand, n1-standard-16) 1,460 hours $776.72 │ └─ Standard provisioned storage (pd-standard) 200 GB $8.00 └─ node_pool[1] - ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 2,920 hours $553.34 + ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 2,920 hours $608.76 └─ Standard provisioned storage (pd-standard) 400 GB $16.00 google_container_cluster.autopilot_with_usage @@ -138,7 +138,7 @@ ├─ Autopilot memory Monthly cost depends on usage: $3.59 per GB └─ Autopilot ephemeral storage Monthly cost depends on usage: $0.040004 per GB - OVERALL TOTAL $29,469.78 + OVERALL TOTAL $30,162.55 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -149,7 +149,7 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $28,961 ┃ $509 ┃ $29,470 ┃ +┃ main ┃ $29,654 ┃ $509 ┃ $30,163 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ Logs: diff --git a/internal/providers/terraform/google/testdata/container_node_pool_test/container_node_pool_test.golden b/internal/providers/terraform/google/testdata/container_node_pool_test/container_node_pool_test.golden index d37840026e5..3093f0d666e 100644 --- a/internal/providers/terraform/google/testdata/container_node_pool_test/container_node_pool_test.golden +++ b/internal/providers/terraform/google/testdata/container_node_pool_test/container_node_pool_test.golden @@ -64,20 +64,20 @@ ├─ Instance usage (Linux/UNIX, on-demand, e2-medium) 5,840 hours $195.67 └─ Standard provisioned storage (pd-standard) 800 GB $32.00 - google_container_node_pool.autoscaling_zonal - ├─ Instance usage (Linux/UNIX, on-demand, e2-medium) 4,380 hours $146.76 - └─ Standard provisioned storage (pd-standard) 600 GB $24.00 - google_container_node_pool.with_preemptible_instance - ├─ Custom instance CPU (Linux/UNIX, preemptible, N1 6 vCPUs) 2,190 hours $110.38 - ├─ Custom Instance RAM (Linux/UNIX, preemptible, N1 20 GB) 2,190 hours $47.13 + ├─ Custom instance CPU (Linux/UNIX, preemptible, N1 6 vCPUs) 2,190 hours $121.41 + ├─ Custom Instance RAM (Linux/UNIX, preemptible, N1 20 GB) 2,190 hours $51.86 └─ Standard provisioned storage (pd-standard) 300 GB $12.00 google_container_node_pool.with_spot_instance - ├─ Custom instance CPU (Linux/UNIX, preemptible, N1 6 vCPUs) 2,190 hours $110.38 - ├─ Custom Instance RAM (Linux/UNIX, preemptible, N1 20 GB) 2,190 hours $47.13 + ├─ Custom instance CPU (Linux/UNIX, preemptible, N1 6 vCPUs) 2,190 hours $121.41 + ├─ Custom Instance RAM (Linux/UNIX, preemptible, N1 20 GB) 2,190 hours $51.86 └─ Standard provisioned storage (pd-standard) 300 GB $12.00 + google_container_node_pool.autoscaling_zonal + ├─ Instance usage (Linux/UNIX, on-demand, e2-medium) 4,380 hours $146.76 + └─ Standard provisioned storage (pd-standard) 600 GB $24.00 + google_container_cluster.default_regional ├─ Cluster management fee 730 hours $73.00 └─ default_pool @@ -122,7 +122,7 @@ ├─ Instance usage (Linux/UNIX, on-demand, e2-medium) 1,460 hours $48.92 └─ Standard provisioned storage (pd-standard) 200 GB $8.00 - OVERALL TOTAL $28,044.47 + OVERALL TOTAL $28,076.01 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -133,5 +133,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $28,044 ┃ - ┃ $28,044 ┃ +┃ main ┃ $28,076 ┃ - ┃ $28,076 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file From 619edeaf6beff070ec2579f64580daf49674d473 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Fri, 25 Oct 2024 12:23:00 +0100 Subject: [PATCH 26/81] fix: support "**" in excluded_dirs --- .../excluded_dirs_glob/expected.golden | 10 ++++++++++ .../excluded_dirs_glob/infracost.yml.tmpl | 4 ++++ .../generate/excluded_dirs_glob/tree.txt | 10 ++++++++++ internal/hcl/project_locator.go | 18 ++++++++++-------- 4 files changed, 34 insertions(+), 8 deletions(-) create mode 100644 cmd/infracost/testdata/generate/excluded_dirs_glob/expected.golden create mode 100644 cmd/infracost/testdata/generate/excluded_dirs_glob/infracost.yml.tmpl create mode 100644 cmd/infracost/testdata/generate/excluded_dirs_glob/tree.txt diff --git a/cmd/infracost/testdata/generate/excluded_dirs_glob/expected.golden b/cmd/infracost/testdata/generate/excluded_dirs_glob/expected.golden new file mode 100644 index 00000000000..f95de981a09 --- /dev/null +++ b/cmd/infracost/testdata/generate/excluded_dirs_glob/expected.golden @@ -0,0 +1,10 @@ +version: 0.1 + +projects: + - path: apps/bar + name: apps-bar + skip_autodetect: true + - path: apps/foo + name: apps-foo + skip_autodetect: true + diff --git a/cmd/infracost/testdata/generate/excluded_dirs_glob/infracost.yml.tmpl b/cmd/infracost/testdata/generate/excluded_dirs_glob/infracost.yml.tmpl new file mode 100644 index 00000000000..ab612fdb6f1 --- /dev/null +++ b/cmd/infracost/testdata/generate/excluded_dirs_glob/infracost.yml.tmpl @@ -0,0 +1,4 @@ +version: 0.1 +autodetect: + exclude_dirs: + - "apps/foo/**" diff --git a/cmd/infracost/testdata/generate/excluded_dirs_glob/tree.txt b/cmd/infracost/testdata/generate/excluded_dirs_glob/tree.txt new file mode 100644 index 00000000000..3da3bfaceec --- /dev/null +++ b/cmd/infracost/testdata/generate/excluded_dirs_glob/tree.txt @@ -0,0 +1,10 @@ +. +└── apps/ + ├── bar/ + │ └── main.tf + └── foo/ + ├── bar/ + │ ├── main.tf + │ └── bat/ + │ └── main.tf + └── main.tf diff --git a/internal/hcl/project_locator.go b/internal/hcl/project_locator.go index ff5c9289d4c..27df207a0cd 100644 --- a/internal/hcl/project_locator.go +++ b/internal/hcl/project_locator.go @@ -1823,7 +1823,7 @@ func (p *ProjectLocator) processFile(name string) bool { func buildDirMatcher(dirs []string, fullPath string) func(string) bool { var rawMatches []string - globMatches := make(map[string]struct{}) + globMatchers := []glob.Glob{} for _, dir := range dirs { var absoluteDir string @@ -1837,17 +1837,19 @@ func buildDirMatcher(dirs []string, fullPath string) func(string) bool { absoluteDir = filepath.Join(fullPath, dir) } - globs, err := filepath.Glob(absoluteDir) - if err == nil { - for _, m := range globs { - globMatches[m] = struct{}{} - } + g, err := glob.Compile(absoluteDir) + if err != nil { + continue } + + globMatchers = append(globMatchers, g) } return func(dir string) bool { - if _, ok := globMatches[dir]; ok { - return true + for _, g := range globMatchers { + if g.Match(dir) { + return true + } } base := filepath.Base(dir) From e6683f5c4de22ed09f5358d0523946d707e4ee3e Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Fri, 25 Oct 2024 13:15:26 +0100 Subject: [PATCH 27/81] chore: update golden tests --- .../breakdown_format_json_with_tags_aft_module.golden | 9 +++++++++ .../breakdown_format_json_with_tags_aft_module/main.tf | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/cmd/infracost/testdata/breakdown_format_json_with_tags_aft_module/breakdown_format_json_with_tags_aft_module.golden b/cmd/infracost/testdata/breakdown_format_json_with_tags_aft_module/breakdown_format_json_with_tags_aft_module.golden index 453bbde9f36..bc405be5bd3 100644 --- a/cmd/infracost/testdata/breakdown_format_json_with_tags_aft_module/breakdown_format_json_with_tags_aft_module.golden +++ b/cmd/infracost/testdata/breakdown_format_json_with_tags_aft_module/breakdown_format_json_with_tags_aft_module.golden @@ -1012,6 +1012,15 @@ "managed_by": "AFT" } }, + { + "defaultTags": { + "managed_by": "AFT" + }, + "name": "module.aft.module.aft_ssm_parameters.aws_ssm_parameter.gitlab_selfmanaged_url", + "tags": { + "managed_by": "AFT" + } + }, { "defaultTags": { "managed_by": "AFT" diff --git a/cmd/infracost/testdata/breakdown_format_json_with_tags_aft_module/main.tf b/cmd/infracost/testdata/breakdown_format_json_with_tags_aft_module/main.tf index e1fc24bbd57..54698d53222 100644 --- a/cmd/infracost/testdata/breakdown_format_json_with_tags_aft_module/main.tf +++ b/cmd/infracost/testdata/breakdown_format_json_with_tags_aft_module/main.tf @@ -14,7 +14,7 @@ provider "aws" { } module "aft" { - source = "github.com/aws-ia/terraform-aws-control_tower_account_factory" + source = "github.com/aws-ia/terraform-aws-control_tower_account_factory?ref=1.13.2" # Required Variables ct_management_account_id = "1234" log_archive_account_id = "1234" From b0605d729d6d3477f1dfe5241f49b7126d102313 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Mon, 28 Oct 2024 17:00:48 +0000 Subject: [PATCH 28/81] enhance: lookup Spacelift stack by full text search We won't always have the exact same project name as the stack in Spacelift, so this tries to match using the fulltext search. It will only match if there is exactly one result. --- internal/hcl/remote_variables_loader.go | 43 ++++++------- internal/hcl/remote_variables_loader_test.go | 63 +++++++++++++++++++- 2 files changed, 82 insertions(+), 24 deletions(-) diff --git a/internal/hcl/remote_variables_loader.go b/internal/hcl/remote_variables_loader.go index b4b510ee505..493c2c31e6f 100644 --- a/internal/hcl/remote_variables_loader.go +++ b/internal/hcl/remote_variables_loader.go @@ -409,24 +409,29 @@ func (s *SpaceliftRemoteVariableLoader) Load(options RemoteVarLoaderOptions) (ma // in future we should get all stacks for the remote name and then // dynamically create projects out of the stacks returned. stacks, err := s.getStacks(context.Background(), &getStackOptions{ - count: 1, + count: 2, // We only want 1 stack, but want to check if there's more than 1 repositoryName: s.Metadata.Remote.Name, - name: options.EnvName, + fullTextSearch: options.EnvName, }) if err != nil { return nil, fmt.Errorf("could not get stacks: %w", err) } - var stackEnvs []stackConfig - for _, s := range stacks { - if s.Name == options.EnvName { - stackEnvs = s.Config - break - } + if len(stacks) == 0 { + logging.Logger.Debug().Msg("no stack found, skipping Spacelift remote variable loading") + return nil, nil + } + + if len(stacks) > 1 { + logging.Logger.Debug().Msg("more than one stack found, skipping Spacelift remote variable loading") + return nil, nil } + logging.Logger.Debug().Msgf("found stack %s for environment %s", stacks[0].Name, options.EnvName) + stackEnvs := stacks[0].Config + if len(stackEnvs) == 0 { - logging.Logger.Trace().Msg("no stack environments found, skipping Spacelift remote variable loading") + logging.Logger.Debug().Msg("no stack environment variables found, skipping Spacelift remote variable loading") return nil, nil } @@ -453,7 +458,7 @@ type getStackOptions struct { count int32 repositoryName string - name string + fullTextSearch string } func (s *SpaceliftRemoteVariableLoader) getStacks(ctx context.Context, p *getStackOptions) ([]stack, error) { @@ -479,20 +484,16 @@ func (s *SpaceliftRemoteVariableLoader) getStacks(ctx context.Context, p *getSta }) } - if p.name != "" { - conditions = append(conditions, structs.QueryPredicate{ - Field: graphql.String("name"), - Constraint: structs.QueryFieldConstraint{ - StringMatches: &[]graphql.String{graphql.String(p.name)}, - }, - }) + input := structs.SearchInput{ + First: graphql.NewInt(graphql.Int(p.count)), + Predicates: &conditions, + } + if p.fullTextSearch != "" { + input.FullTextSearch = graphql.NewString(graphql.String(p.fullTextSearch)) } - variables := map[string]interface{}{"input": structs.SearchInput{ - First: graphql.NewInt(graphql.Int(p.count)), - Predicates: &conditions, - }} + variables := map[string]interface{}{"input": input} if err := s.Client.Query( ctx, diff --git a/internal/hcl/remote_variables_loader_test.go b/internal/hcl/remote_variables_loader_test.go index cb4894952f2..cacfc8b9980 100644 --- a/internal/hcl/remote_variables_loader_test.go +++ b/internal/hcl/remote_variables_loader_test.go @@ -57,7 +57,7 @@ func TestSpaceliftRemoteVariableLoader_Load(t *testing.T) { assert.Equal(t, "POST", r.Method) s, err := io.ReadAll(r.Body) assert.NoError(t, err) - assert.JSONEq(t, `{"query":"query($input:SearchInput!){searchStacks(input: $input){edges{node{id,name,config{id,value}}},pageInfo{endCursor,hasNextPage,hasPreviousPage}}}","variables":{"input":{"first":1,"after":null,"fullTextSearch":null,"predicates":[{"field":"repository","constraint":{"booleanEquals":null,"enumEquals":null,"stringMatches":["test/test"]}},{"field":"name","constraint":{"booleanEquals":null,"enumEquals":null,"stringMatches":["dev"]}}],"orderBy":null}}}`, string(s)) + assert.JSONEq(t, `{"query":"query($input:SearchInput!){searchStacks(input: $input){edges{node{id,name,config{id,value}}},pageInfo{endCursor,hasNextPage,hasPreviousPage}}}","variables":{"input":{"first":2,"after":null,"fullTextSearch":"dev","predicates":[{"field":"repository","constraint":{"booleanEquals":null,"enumEquals":null,"stringMatches":["test/test"]}}],"orderBy":null}}}`, string(s)) assert.Equal(t, "Bearer test", r.Header.Get("Authorization")) _, err = w.Write([]byte(`{"data":{"searchStacks":{"edges":[{"node":{"id":"dev","name":"dev","config":[{"id":"test","value":"1"},{"id":"test2","value":"foo"}]}}],"pageInfo":{"endCursor":"MDFKNEtQMzVETjFYRDcxNTY1UkJCMzBITUQ=","hasNextPage":true,"hasPreviousPage":false}}}}`)) @@ -87,14 +87,71 @@ func TestSpaceliftRemoteVariableLoader_Load(t *testing.T) { assert.Equal(t, "POST", r.Method) s, err := io.ReadAll(r.Body) assert.NoError(t, err) - assert.JSONEq(t, `{"query":"query($input:SearchInput!){searchStacks(input: $input){edges{node{id,name,config{id,value}}},pageInfo{endCursor,hasNextPage,hasPreviousPage}}}","variables":{"input":{"first":1,"after":null,"fullTextSearch":null,"predicates":[{"field":"repository","constraint":{"booleanEquals":null,"enumEquals":null,"stringMatches":["test/test"]}},{"field":"name","constraint":{"booleanEquals":null,"enumEquals":null,"stringMatches":["dev"]}}],"orderBy":null}}}`, string(s)) - assert.Equal(t, "Bearer test", r.Header.Get("Authorization")) + assert.JSONEq(t, `{"query":"query($input:SearchInput!){searchStacks(input: $input){edges{node{id,name,config{id,value}}},pageInfo{endCursor,hasNextPage,hasPreviousPage}}}","variables":{"input":{"first":2,"after":null,"fullTextSearch":"dev","predicates":[{"field":"repository","constraint":{"booleanEquals":null,"enumEquals":null,"stringMatches":["test/test"]}}],"orderBy":null}}}`, string(s)) _, err = w.Write([]byte(`{"data":{"searchStacks":{"edges":[{"node":{"id":"dev","name":"dev","config":[]}}],"pageInfo":{"endCursor":"MDFKNEtQMzVETjFYRDcxNTY1UkJCMzBITUQ=","hasNextPage":true,"hasPreviousPage":false}}}}`)) assert.NoError(t, err) } }, }, + { + name: "returns empty map if no stack is found", + fields: fields{ + Metadata: vcs.Metadata{ + Remote: vcs.Remote{ + Name: "test/test", + }, + }, + }, + args: args{ + options: RemoteVarLoaderOptions{ + EnvName: "dev", + }, + }, + want: nil, + wantErr: assert.NoError, + testServerFunc: func(t *testing.T) http.HandlerFunc { + // test that the /graqphl endpoint has been called and the correct query has been sent + return func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "POST", r.Method) + s, err := io.ReadAll(r.Body) + assert.NoError(t, err) + assert.JSONEq(t, `{"query":"query($input:SearchInput!){searchStacks(input: $input){edges{node{id,name,config{id,value}}},pageInfo{endCursor,hasNextPage,hasPreviousPage}}}","variables":{"input":{"first":2,"after":null,"fullTextSearch":"dev","predicates":[{"field":"repository","constraint":{"booleanEquals":null,"enumEquals":null,"stringMatches":["test/test"]}}],"orderBy":null}}}`, string(s)) + + _, err = w.Write([]byte(`{"data":{"searchStacks":{"edges":[],"pageInfo":{"endCursor":"MDFKNEtQMzVETjFYRDcxNTY1UkJCMzBITUQ=","hasNextPage":true,"hasPreviousPage":false}}}}`)) + assert.NoError(t, err) + } + }, + }, + { + name: "returns empty map if more than one stack is found", + fields: fields{ + Metadata: vcs.Metadata{ + Remote: vcs.Remote{ + Name: "test/test", + }, + }, + }, + args: args{ + options: RemoteVarLoaderOptions{ + EnvName: "dev", + }, + }, + want: nil, + wantErr: assert.NoError, + testServerFunc: func(t *testing.T) http.HandlerFunc { + // test that the /graqphl endpoint has been called and the correct query has been sent + return func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "POST", r.Method) + s, err := io.ReadAll(r.Body) + assert.NoError(t, err) + assert.JSONEq(t, `{"query":"query($input:SearchInput!){searchStacks(input: $input){edges{node{id,name,config{id,value}}},pageInfo{endCursor,hasNextPage,hasPreviousPage}}}","variables":{"input":{"first":2,"after":null,"fullTextSearch":"dev","predicates":[{"field":"repository","constraint":{"booleanEquals":null,"enumEquals":null,"stringMatches":["test/test"]}}],"orderBy":null}}}`, string(s)) + + _, err = w.Write([]byte(`{"data":{"searchStacks":{"edges":[{"node":{"id":"dev-a","name":"dev-a","config":[]}},{"node":{"id":"dev-b","name":"dev-b","config":[]}}],"pageInfo":{"endCursor":"MDFKNEtQMzVETjFYRDcxNTY1UkJCMzBITUQ=","hasNextPage":true,"hasPreviousPage":false}}}}`)) + assert.NoError(t, err) + } + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 8b8b8212e538a00e05bb3e78e2958c72ca3ca3c4 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Tue, 29 Oct 2024 13:58:43 +0000 Subject: [PATCH 29/81] enhance: support ? wildcards in `env_names` --- .../wildcard_env_names/expected.golden | 32 +++++++++++++++++++ .../wildcard_env_names/infracost.yml.tmpl | 1 + .../generate/wildcard_env_names/tree.txt | 3 ++ internal/hcl/project_locator.go | 5 +-- 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/cmd/infracost/testdata/generate/wildcard_env_names/expected.golden b/cmd/infracost/testdata/generate/wildcard_env_names/expected.golden index 1b5e604b970..787b1e6cb02 100755 --- a/cmd/infracost/testdata/generate/wildcard_env_names/expected.golden +++ b/cmd/infracost/testdata/generate/wildcard_env_names/expected.golden @@ -4,49 +4,81 @@ projects: - path: terraform name: terraform-conf-dev-foo terraform_var_files: + - env/other-uat.tfvars - env/conf-dev-foo.tfvars skip_autodetect: true dependency_paths: - terraform/** - terraform/env/conf-dev-foo.tfvars + - terraform/env/other-uat.tfvars - path: terraform name: terraform-conf-prod-foo terraform_var_files: + - env/other-uat.tfvars - env/conf-prod-foo.tfvars skip_autodetect: true dependency_paths: - terraform/** - terraform/env/conf-prod-foo.tfvars + - terraform/env/other-uat.tfvars - path: terraform name: terraform-dev terraform_var_files: + - env/other-uat.tfvars - env/dev.tfvars skip_autodetect: true dependency_paths: - terraform/** - terraform/env/dev.tfvars + - terraform/env/other-uat.tfvars - path: terraform name: terraform-ops-dev terraform_var_files: + - env/other-uat.tfvars - env/ops-dev.tfvars skip_autodetect: true dependency_paths: - terraform/** - terraform/env/ops-dev.tfvars + - terraform/env/other-uat.tfvars - path: terraform name: terraform-ops-prod-bar terraform_var_files: + - env/other-uat.tfvars - env/ops-prod-bar.tfvars skip_autodetect: true dependency_paths: - terraform/** - terraform/env/ops-prod-bar.tfvars + - terraform/env/other-uat.tfvars - path: terraform name: terraform-ops-prod-foo terraform_var_files: + - env/other-uat.tfvars - env/ops-prod-foo.tfvars skip_autodetect: true dependency_paths: - terraform/** - terraform/env/ops-prod-foo.tfvars + - terraform/env/other-uat.tfvars + - path: terraform + name: terraform-uk-uat + terraform_var_files: + - env/other-uat.tfvars + - env/uk-uat.tfvars + skip_autodetect: true + dependency_paths: + - terraform/** + - terraform/env/other-uat.tfvars + - terraform/env/uk-uat.tfvars + - path: terraform + name: terraform-us-uat + terraform_var_files: + - env/other-uat.tfvars + - env/us-uat.tfvars + skip_autodetect: true + dependency_paths: + - terraform/** + - terraform/env/other-uat.tfvars + - terraform/env/us-uat.tfvars diff --git a/cmd/infracost/testdata/generate/wildcard_env_names/infracost.yml.tmpl b/cmd/infracost/testdata/generate/wildcard_env_names/infracost.yml.tmpl index 0d1ce52afdd..afc149f3cf4 100644 --- a/cmd/infracost/testdata/generate/wildcard_env_names/infracost.yml.tmpl +++ b/cmd/infracost/testdata/generate/wildcard_env_names/infracost.yml.tmpl @@ -4,4 +4,5 @@ autodetect: - ops-* - conf-* - dev + - ??-uat diff --git a/cmd/infracost/testdata/generate/wildcard_env_names/tree.txt b/cmd/infracost/testdata/generate/wildcard_env_names/tree.txt index b016de56b3f..cf3e0f0d5c7 100644 --- a/cmd/infracost/testdata/generate/wildcard_env_names/tree.txt +++ b/cmd/infracost/testdata/generate/wildcard_env_names/tree.txt @@ -6,5 +6,8 @@ │ ├── ops-dev.tfvars │ ├── conf-dev-foo.tfvars │ ├── dev.tfvars + │ ├── uk-uat.tfvars + │ ├── us-uat.tfvars + │ ├── other-uat.tfvars │ └── conf-prod-foo.tfvars └── main.tf diff --git a/internal/hcl/project_locator.go b/internal/hcl/project_locator.go index 27df207a0cd..d208db00a1c 100644 --- a/internal/hcl/project_locator.go +++ b/internal/hcl/project_locator.go @@ -107,8 +107,8 @@ func CreateEnvFileMatcher(names []string, extensions []string) *EnvFileMatcher { // will create separate envs for dev-staging and dev-legacy. We don't want these // wildcards to appear in the envNames list as this will create unwanted env // grouping. - if strings.Contains(name, "*") { - wildcards = append(wildcards, name) + if strings.Contains(name, "*") || strings.Contains(name, "?") { + wildcards = append(wildcards, strings.ToLower(name)) continue } @@ -170,6 +170,7 @@ func (e *EnvFileMatcher) IsGlobalVarFile(file string) bool { // IsEnvName checks if the var file is an environment specific var file. func (e *EnvFileMatcher) IsEnvName(file string) bool { clean := e.clean(file) + _, ok := e.envLookup[clean] if ok { return true From 045395438b0c8e66acc1eb8ec9e1f378334862cd Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Tue, 29 Oct 2024 14:01:25 +0000 Subject: [PATCH 30/81] feat: add `lower` function to config template --- internal/config/template/parser.go | 6 ++++++ internal/hcl/project_locator.go | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/config/template/parser.go b/internal/config/template/parser.go index 0005d1b1cfd..dd2cd3e6aa0 100644 --- a/internal/config/template/parser.go +++ b/internal/config/template/parser.go @@ -62,6 +62,7 @@ func NewParser(repoDir string, variables Variables) *Parser { "base": p.base, "stem": p.stem, "ext": p.ext, + "lower": p.lower, "startsWith": p.startsWith, "endsWith": p.endsWith, "contains": p.contains, @@ -145,6 +146,11 @@ func (p *Parser) ext(path string) string { return filepath.Ext(path) } +// lower returns a copy of the string s with all Unicode letters mapped to their lower case. +func (p *Parser) lower(s string) string { + return strings.ToLower(s) +} + // startsWith tests whether the string s begins with prefix. func (p *Parser) startsWith(s, prefix string) bool { return strings.HasPrefix(s, prefix) diff --git a/internal/hcl/project_locator.go b/internal/hcl/project_locator.go index d208db00a1c..03b3a666efb 100644 --- a/internal/hcl/project_locator.go +++ b/internal/hcl/project_locator.go @@ -170,7 +170,6 @@ func (e *EnvFileMatcher) IsGlobalVarFile(file string) bool { // IsEnvName checks if the var file is an environment specific var file. func (e *EnvFileMatcher) IsEnvName(file string) bool { clean := e.clean(file) - _, ok := e.envLookup[clean] if ok { return true From df43fc8355674597444b3f61950372af78e35346 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Wed, 30 Oct 2024 18:26:12 +0000 Subject: [PATCH 31/81] enhance: add `splitList` config file function Naming this so it's consistent with sprig --- internal/config/template/parser.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/config/template/parser.go b/internal/config/template/parser.go index dd2cd3e6aa0..fa0d164429f 100644 --- a/internal/config/template/parser.go +++ b/internal/config/template/parser.go @@ -66,6 +66,7 @@ func NewParser(repoDir string, variables Variables) *Parser { "startsWith": p.startsWith, "endsWith": p.endsWith, "contains": p.contains, + "splitList": p.splitList, "trimPrefix": p.trimPrefix, "trimSuffix": p.trimSuffix, "replace": p.replace, @@ -166,6 +167,11 @@ func (p *Parser) contains(s, substr string) bool { return strings.Contains(s, substr) } +// splitList splits the string s into a slice of substrings separated by sep. +func (p *Parser) splitList(sep, s string) []string { + return strings.Split(s, sep) +} + // trimPrefix returns s without the provided prefix string. func (p *Parser) trimPrefix(s, prefix string) string { return strings.TrimPrefix(s, prefix) From 2ff32d8434bff07ed79d154fe545aed1a4311b97 Mon Sep 17 00:00:00 2001 From: Liam Galvin Date: Thu, 31 Oct 2024 17:17:27 +0000 Subject: [PATCH 32/81] fix: Remove .git from repo names in terraformModulePath (#3236) --- internal/apiclient/policy.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/apiclient/policy.go b/internal/apiclient/policy.go index 66f39d3495c..a196302e1f9 100644 --- a/internal/apiclient/policy.go +++ b/internal/apiclient/policy.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "fmt" "sort" + "strings" "sync" "github.com/hashicorp/go-retryablehttp" @@ -49,6 +50,9 @@ func (c *PolicyAPIClient) UploadPolicyData(project *schema.Project, rds, pastRds project.Metadata = &schema.ProjectMetadata{} } + // remove .git suffix from the module path + project.Metadata.TerraformModulePath = strings.ReplaceAll(project.Metadata.TerraformModulePath, ".git/", "/") + err := c.fetchAllowList() if err != nil { return err From 9093ec4459db0357e27fddf7c4ffa7f480ca7197 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Thu, 31 Oct 2024 19:37:48 +0000 Subject: [PATCH 33/81] fix: fix Spacelift var loading MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Update the Spacelift remote variable loader to load in variables from all 3 of the locations: - Environment - Spacelift environment - Attached contexts. We should look for any key starting with `TF_VAR_` 2. Update the Spacelift remote variable loader to better match on stack name. I think the following logic would work: 1. Match the repository name exactly 2. Match the projectRoot exactly based on the path of the module 3. Query for multiple results 4. If the project has a `moduleSuffix` then match on that using what is after the `:` in the project name. If there’s no `moduleSuffix`, then skip this step. 5. Log a warning if there’s more than one match or no matches --- internal/hcl/parser.go | 22 +- internal/hcl/remote_variables_loader.go | 104 ++++- internal/hcl/remote_variables_loader_test.go | 451 ++++++++++++++++++- internal/providers/terraform/hcl_provider.go | 16 +- 4 files changed, 533 insertions(+), 60 deletions(-) diff --git a/internal/hcl/parser.go b/internal/hcl/parser.go index 2961749823f..0d740b22d17 100644 --- a/internal/hcl/parser.go +++ b/internal/hcl/parser.go @@ -275,6 +275,14 @@ func OptionGraphEvaluator() Option { } } +// OptionWithProjectName sets the project name for the parser. +// This is used if the project name has been explicitly set by the user or the autodetection +func OptionWithProjectName(name string) Option { + return func(p *Parser) { + p.projectName = name + } +} + type DetectedProject interface { ProjectName() string EnvName() string @@ -288,6 +296,7 @@ type DetectedProject interface { type Parser struct { startingPath string detectedProjectPath string + projectName string tfEnvVars map[string]cty.Value tfvarsPaths []string inputVars map[string]cty.Value @@ -553,10 +562,16 @@ func (p *Parser) RelativePath() string { // ProjectName generates a name for the project that can be used // in the Infracost config file. func (p *Parser) ProjectName() string { + if p.projectName != "" { + return p.projectName + } + name := config.CleanProjectName(p.RelativePath()) if p.moduleSuffix != "" { name = fmt.Sprintf("%s-%s", name, p.moduleSuffix) + } else if p.workspaceName != "" && p.workspaceName != defaultTerraformWorkspaceName { + name = fmt.Sprintf("%s-%s", name, p.workspaceName) } return name @@ -568,7 +583,7 @@ func (p *Parser) EnvName() string { return p.moduleSuffix } - return p.ProjectName() + return p.envMatcher.EnvName(p.ProjectName()) } // TerraformVarFiles returns the list of terraform var files that the parser @@ -626,8 +641,9 @@ func (p *Parser) loadVars(blocks Blocks, filenames []string) (map[string]cty.Val if p.remoteVariableLoaders != nil { for _, loader := range p.remoteVariableLoaders { remoteVars, err := loader.Load(RemoteVarLoaderOptions{ - Blocks: blocks, - EnvName: p.EnvName(), + Blocks: blocks, + ModulePath: p.RelativePath(), + Environment: p.EnvName(), }) if err != nil { p.logger.Debug().Msgf("could not load vars from Terraform Cloud: %s", err) diff --git a/internal/hcl/remote_variables_loader.go b/internal/hcl/remote_variables_loader.go index 493c2c31e6f..0a83ab468c8 100644 --- a/internal/hcl/remote_variables_loader.go +++ b/internal/hcl/remote_variables_loader.go @@ -17,6 +17,7 @@ import ( spaceliftSession "github.com/spacelift-io/spacectl/client/session" "github.com/spacelift-io/spacectl/client/structs" "github.com/zclconf/go-cty/cty" + ctyJson "github.com/zclconf/go-cty/cty/json" "github.com/infracost/infracost/internal/extclient" "github.com/infracost/infracost/internal/logging" @@ -120,8 +121,9 @@ func NewTFCRemoteVariablesLoader(client *extclient.AuthedAPIClient, localWorkspa } type RemoteVarLoaderOptions struct { - Blocks Blocks - EnvName string + Blocks Blocks + ModulePath string + Environment string } // Load fetches remote variables if terraform block contains organization and @@ -400,8 +402,8 @@ func NewSpaceliftRemoteVariableLoader(metadata vcs.Metadata, apiKeyEndpoint, api // Load fetches remote variables from Spacelift by querying the stacks for the // provided environment name and remote name. func (s *SpaceliftRemoteVariableLoader) Load(options RemoteVarLoaderOptions) (map[string]cty.Value, error) { - if options.EnvName == "" { - logging.Logger.Trace().Msg("no environment name provided, skipping Spacelift remote variable loading") + if options.ModulePath == "" { + logging.Logger.Trace().Msg("no module path provided, skipping Spacelift remote variable loading") return nil, nil } @@ -409,44 +411,83 @@ func (s *SpaceliftRemoteVariableLoader) Load(options RemoteVarLoaderOptions) (ma // in future we should get all stacks for the remote name and then // dynamically create projects out of the stacks returned. stacks, err := s.getStacks(context.Background(), &getStackOptions{ - count: 2, // We only want 1 stack, but want to check if there's more than 1 + count: 10, // We only want 1 stack, but want to filter by the module suffix repositoryName: s.Metadata.Remote.Name, - fullTextSearch: options.EnvName, + projectRoot: options.ModulePath, }) if err != nil { - return nil, fmt.Errorf("could not get stacks: %w", err) + return nil, fmt.Errorf("could not get Spacelift stacks: %w", err) } if len(stacks) == 0 { - logging.Logger.Debug().Msg("no stack found, skipping Spacelift remote variable loading") + logging.Logger.Debug().Msgf("no Spacelift stack found for module path %q", options.ModulePath) return nil, nil } - if len(stacks) > 1 { - logging.Logger.Debug().Msg("more than one stack found, skipping Spacelift remote variable loading") - return nil, nil + // If there is a module suffix, filter the stacks by it + if options.Environment != "" { + var filteredStacks []stack + for _, stack := range stacks { + if strings.HasSuffix(stack.Name, fmt.Sprintf(":%s", options.Environment)) { + filteredStacks = append(filteredStacks, stack) + } + } + stacks = filteredStacks } - logging.Logger.Debug().Msgf("found stack %s for environment %s", stacks[0].Name, options.EnvName) - stackEnvs := stacks[0].Config + if len(stacks) == 0 { + logging.Logger.Warn().Msgf("no Spacelift stack found for module path %q with environment %q", options.ModulePath, options.Environment) + return nil, nil + } - if len(stackEnvs) == 0 { - logging.Logger.Debug().Msg("no stack environment variables found, skipping Spacelift remote variable loading") + if len(stacks) > 1 { + logging.Logger.Warn().Msgf("found multiple Spacelift stacks for module path %q with environment %q, skipping Spacelift remote variable loading", options.ModulePath, options.Environment) return nil, nil } + logging.Logger.Debug().Msgf("found Spacelift stack %q for module path %q with environment: %q", stacks[0].Name, options.ModulePath, options.Environment) + vars := map[string]cty.Value{} - for _, env := range stackEnvs { - vars[env.ID] = cty.StringVal(env.Value) + + // Spacelift precedence is runtime config > config > attached contexts + for _, env := range stacks[0].AttachedContexts { + if strings.HasPrefix(env.ID, "TF_VAR_") { + vars[strings.TrimPrefix(env.ID, "TF_VAR_")] = cty.StringVal(env.ContextName) + } + } + + for _, env := range stacks[0].Config { + if strings.HasPrefix(env.ID, "TF_VAR_") { + vars[strings.TrimPrefix(env.ID, "TF_VAR_")] = cty.StringVal(env.Value) + } + } + + for _, env := range stacks[0].RuntimeConfig { + if strings.HasPrefix(env.Element.ID, "TF_VAR_") { + vars[strings.TrimPrefix(env.Element.ID, "TF_VAR_")] = cty.StringVal(env.Element.Value) + } + } + + logging.Logger.Debug().Msgf("loaded %d Spacelift remote variables", len(vars)) + + // Only marshal and log the variables if the log level is trace + // Otherwise we want to skip the marshalling for performance reasons + if logging.Logger.GetLevel() == zerolog.TraceLevel { + s := ctyJson.SimpleJSONValue{Value: cty.ObjectVal(vars)} + b, _ := s.MarshalJSON() + logging.Logger.Trace().Msgf("Spacelift remote variables: %v", string(b)) } return vars, nil } type stack struct { - ID string `graphql:"id" json:"id,omitempty"` - Name string `graphql:"name" json:"name,omitempty"` - Config []stackConfig `graphql:"config" json:"config,omitempty"` + ID string `graphql:"id" json:"id,omitempty"` + Name string `graphql:"name" json:"name,omitempty"` + ProjectRoot string `graphql:"projectRoot" json:"projectRoot,omitempty"` + Config []stackConfig `graphql:"config" json:"config,omitempty"` + RuntimeConfig []runtimeConfig `graphql:"runtimeConfig" json:"runtimeConfig,omitempty"` + AttachedContexts []attachedContext `graphql:"attachedContexts" json:"attachedContexts,omitempty"` } type stackConfig struct { @@ -454,11 +495,20 @@ type stackConfig struct { Value string `graphql:"value" json:"value,omitempty"` } +type runtimeConfig struct { + Element stackConfig `graphql:"element" json:"element,omitempty"` +} + +type attachedContext struct { + ID string `graphql:"id" json:"id,omitempty"` + ContextName string `graphql:"contextName" json:"contextName,omitempty"` +} + type getStackOptions struct { count int32 repositoryName string - fullTextSearch string + projectRoot string } func (s *SpaceliftRemoteVariableLoader) getStacks(ctx context.Context, p *getStackOptions) ([]stack, error) { @@ -483,16 +533,20 @@ func (s *SpaceliftRemoteVariableLoader) getStacks(ctx context.Context, p *getSta }, }) } + if p.projectRoot != "" { + conditions = append(conditions, structs.QueryPredicate{ + Field: graphql.String("projectRoot"), + Constraint: structs.QueryFieldConstraint{ + StringMatches: &[]graphql.String{graphql.String(p.projectRoot)}, + }, + }) + } input := structs.SearchInput{ First: graphql.NewInt(graphql.Int(p.count)), Predicates: &conditions, } - if p.fullTextSearch != "" { - input.FullTextSearch = graphql.NewString(graphql.String(p.fullTextSearch)) - } - variables := map[string]interface{}{"input": input} if err := s.Client.Query( diff --git a/internal/hcl/remote_variables_loader_test.go b/internal/hcl/remote_variables_loader_test.go index cacfc8b9980..70f947e8f86 100644 --- a/internal/hcl/remote_variables_loader_test.go +++ b/internal/hcl/remote_variables_loader_test.go @@ -1,6 +1,7 @@ package hcl import ( + "bytes" "context" "fmt" "io" @@ -9,11 +10,13 @@ import ( "sync" "testing" + "github.com/rs/zerolog" "github.com/spacelift-io/spacectl/client" "github.com/spacelift-io/spacectl/client/session" "github.com/stretchr/testify/assert" "github.com/zclconf/go-cty/cty" + "github.com/infracost/infracost/internal/logging" "github.com/infracost/infracost/internal/vcs" ) @@ -31,9 +34,10 @@ func TestSpaceliftRemoteVariableLoader_Load(t *testing.T) { want map[string]cty.Value wantErr assert.ErrorAssertionFunc testServerFunc func(t *testing.T) http.HandlerFunc + verifyLog func(t *testing.T, log string) }{ { - name: "return variables from spacelift context", + name: "return variables from SpaceLift", fields: fields{ Metadata: vcs.Metadata{ Remote: vcs.Remote{ @@ -43,30 +47,79 @@ func TestSpaceliftRemoteVariableLoader_Load(t *testing.T) { }, args: args{ options: RemoteVarLoaderOptions{ - EnvName: "dev", + ModulePath: "module/path", + Environment: "dev", }, }, want: map[string]cty.Value{ - "test": cty.StringVal("1"), - "test2": cty.StringVal("foo"), + "context_var": cty.StringVal("context_value"), + "config_var": cty.StringVal("config_value"), + "runtime_var": cty.StringVal("runtime_value"), }, wantErr: assert.NoError, testServerFunc: func(t *testing.T) http.HandlerFunc { - // test that the /graqphl endpoint has been called and the correct query has been sent return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") assert.Equal(t, "POST", r.Method) s, err := io.ReadAll(r.Body) assert.NoError(t, err) - assert.JSONEq(t, `{"query":"query($input:SearchInput!){searchStacks(input: $input){edges{node{id,name,config{id,value}}},pageInfo{endCursor,hasNextPage,hasPreviousPage}}}","variables":{"input":{"first":2,"after":null,"fullTextSearch":"dev","predicates":[{"field":"repository","constraint":{"booleanEquals":null,"enumEquals":null,"stringMatches":["test/test"]}}],"orderBy":null}}}`, string(s)) - assert.Equal(t, "Bearer test", r.Header.Get("Authorization")) - _, err = w.Write([]byte(`{"data":{"searchStacks":{"edges":[{"node":{"id":"dev","name":"dev","config":[{"id":"test","value":"1"},{"id":"test2","value":"foo"}]}}],"pageInfo":{"endCursor":"MDFKNEtQMzVETjFYRDcxNTY1UkJCMzBITUQ=","hasNextPage":true,"hasPreviousPage":false}}}}`)) + expectedJSON := `{ + "query": "query($input:SearchInput!){searchStacks(input: $input){edges{node{id,name,projectRoot,config{id,value},runtimeConfig{element{id,value}},attachedContexts{id,contextName}}},pageInfo{endCursor,hasNextPage,hasPreviousPage}}}", + "variables": { + "input": { + "first": 10, + "after": null, + "fullTextSearch": null, + "orderBy": null, + "predicates": [ + {"field": "repository", "constraint": {"stringMatches": ["test/test"], "booleanEquals": null, "enumEquals": null}}, + {"field": "projectRoot", "constraint": {"stringMatches": ["module/path"], "booleanEquals": null, "enumEquals": null}} + ] + } + } + }` + assert.JSONEq(t, expectedJSON, string(s)) + + response := `{ + "data": { + "searchStacks": { + "edges": [ + { + "node": { + "id": "module-path-dev", + "name": "module-path:dev", + "projectRoot": "module/path", + "attachedContexts": [ + {"id": "TF_VAR_context_var", "contextName": "context_value"}, + {"id": "CONTEXT_VAR", "contextName": "non_tf_context_value"} + ], + "config": [ + {"id": "TF_VAR_config_var", "value": "config_value"}, + {"id": "CONFIG_VAR", "value": "non_tf_config_value"} + ], + "runtimeConfig": [ + {"element": {"id": "TF_VAR_runtime_var", "value": "runtime_value"}}, + {"element": {"id": "RUNTIME_VAR", "value": "non_tf_runtime_value"}} + ] + } + } + ], + "pageInfo": { + "endCursor": "MDFKNEtQMzVETjFYRDcxNTY1UkJCMzBITUQ=", + "hasNextPage": false, + "hasPreviousPage": false + } + } + } + }` + _, err = w.Write([]byte(response)) assert.NoError(t, err) } }, }, { - name: "returns empty map if no variables found", + name: "returns the correct variables when multiple stacks match ModulePath but only one has the correct environment", fields: fields{ Metadata: vcs.Metadata{ Remote: vcs.Remote{ @@ -76,26 +129,253 @@ func TestSpaceliftRemoteVariableLoader_Load(t *testing.T) { }, args: args{ options: RemoteVarLoaderOptions{ - EnvName: "dev", + ModulePath: "module/path", + Environment: "prod", + }, + }, + want: map[string]cty.Value{ + "runtime_var_prod": cty.StringVal("runtime_value_prod"), + "config_var_prod": cty.StringVal("config_value_prod"), + "context_var_prod": cty.StringVal("context_value_prod"), + }, + wantErr: assert.NoError, + testServerFunc: func(t *testing.T) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + assert.Equal(t, "POST", r.Method) + s, err := io.ReadAll(r.Body) + assert.NoError(t, err) + + expectedJSON := `{ + "query": "query($input:SearchInput!){searchStacks(input: $input){edges{node{id,name,projectRoot,config{id,value},runtimeConfig{element{id,value}},attachedContexts{id,contextName}}},pageInfo{endCursor,hasNextPage,hasPreviousPage}}}", + "variables": { + "input": { + "first": 10, + "after": null, + "fullTextSearch": null, + "orderBy": null, + "predicates": [ + {"field": "repository", "constraint": {"stringMatches": ["test/test"], "booleanEquals": null, "enumEquals": null}}, + {"field": "projectRoot", "constraint": {"stringMatches": ["module/path"], "booleanEquals": null, "enumEquals": null}} + ] + } + } + }` + assert.JSONEq(t, expectedJSON, string(s)) + + response := `{ + "data": { + "searchStacks": { + "edges": [ + { + "node": { + "id": "module-path-staging", + "name": "module-path-:staging", + "projectRoot": "module/path", + "attachedContexts": [ + {"id": "TF_VAR_context_var_staging", "contextName": "context_value_staging"} + ], + "config": [ + {"id": "TF_VAR_config_var_staging", "value": "config_value_staging"} + ], + "runtimeConfig": [ + {"element": {"id": "TF_VAR_runtime_var_staging", "value": "runtime_value_staging"}} + ] + } + }, + { + "node": { + "id": "module-path-prod", + "name": "module-path:prod", + "projectRoot": "module/path", + "attachedContexts": [ + {"id": "TF_VAR_context_var_prod", "contextName": "context_value_prod"} + ], + "config": [ + {"id": "TF_VAR_config_var_prod", "value": "config_value_prod"} + ], + "runtimeConfig": [ + {"element": {"id": "TF_VAR_runtime_var_prod", "value": "runtime_value_prod"}} + ] + } + } + ], + "pageInfo": { + "endCursor": "MDFKNEtQMzVETjFYRDcxNTY1UkJCMzBITUQ=", + "hasNextPage": false, + "hasPreviousPage": false + } + } + } + }` + _, err = w.Write([]byte(response)) + assert.NoError(t, err) + } + }, + }, + { + name: "returns the correct variables according to the precedence of the config and context values", + fields: fields{ + Metadata: vcs.Metadata{ + Remote: vcs.Remote{ + Name: "test/test", + }, + }, + }, + args: args{ + options: RemoteVarLoaderOptions{ + ModulePath: "module/path", + Environment: "dev", + }, + }, + want: map[string]cty.Value{ + "var1": cty.StringVal("runtime_value"), + "var2": cty.StringVal("config_value"), + }, + wantErr: assert.NoError, + testServerFunc: func(t *testing.T) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + assert.Equal(t, "POST", r.Method) + s, err := io.ReadAll(r.Body) + assert.NoError(t, err) + + expectedJSON := `{ + "query": "query($input:SearchInput!){searchStacks(input: $input){edges{node{id,name,projectRoot,config{id,value},runtimeConfig{element{id,value}},attachedContexts{id,contextName}}},pageInfo{endCursor,hasNextPage,hasPreviousPage}}}", + "variables": { + "input": { + "first": 10, + "after": null, + "fullTextSearch": null, + "orderBy": null, + "predicates": [ + {"field": "repository", "constraint": {"stringMatches": ["test/test"], "booleanEquals": null, "enumEquals": null}}, + {"field": "projectRoot", "constraint": {"stringMatches": ["module/path"], "booleanEquals": null, "enumEquals": null}} + ] + } + } + }` + assert.JSONEq(t, expectedJSON, string(s)) + + response := `{ + "data": { + "searchStacks": { + "edges": [ + { + "node": { + "id": "module-path-dev", + "name": "module-path:dev", + "projectRoot": "module/path", + "attachedContexts": [ + {"id": "TF_VAR_var1", "contextName": "context_value"}, + {"id": "TF_VAR_var2", "contextName": "context_value"} + ], + "config": [ + {"id": "TF_VAR_var1", "value": "config_value"}, + {"id": "TF_VAR_var2", "value": "config_value"} + ], + "runtimeConfig": [ + {"element": {"id": "TF_VAR_var1", "value": "runtime_value"}} + ] + } + } + ], + "pageInfo": { + "endCursor": null, + "hasNextPage": false, + "hasPreviousPage": false + } + } + } + }` + _, err = w.Write([]byte(response)) + assert.NoError(t, err) + } + }, + }, + { + name: "returns nil if no stacks match ModulePath", + fields: fields{ + Metadata: vcs.Metadata{ + Remote: vcs.Remote{ + Name: "test/test", + }, + }, + }, + args: args{ + options: RemoteVarLoaderOptions{ + ModulePath: "nonexistent/path", + Environment: "prod", }, }, want: nil, wantErr: assert.NoError, testServerFunc: func(t *testing.T) http.HandlerFunc { - // test that the /graqphl endpoint has been called and the correct query has been sent return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") assert.Equal(t, "POST", r.Method) s, err := io.ReadAll(r.Body) assert.NoError(t, err) - assert.JSONEq(t, `{"query":"query($input:SearchInput!){searchStacks(input: $input){edges{node{id,name,config{id,value}}},pageInfo{endCursor,hasNextPage,hasPreviousPage}}}","variables":{"input":{"first":2,"after":null,"fullTextSearch":"dev","predicates":[{"field":"repository","constraint":{"booleanEquals":null,"enumEquals":null,"stringMatches":["test/test"]}}],"orderBy":null}}}`, string(s)) - _, err = w.Write([]byte(`{"data":{"searchStacks":{"edges":[{"node":{"id":"dev","name":"dev","config":[]}}],"pageInfo":{"endCursor":"MDFKNEtQMzVETjFYRDcxNTY1UkJCMzBITUQ=","hasNextPage":true,"hasPreviousPage":false}}}}`)) + expectedJSON := `{ + "query": "query($input:SearchInput!){searchStacks(input: $input){edges{node{id,name,projectRoot,config{id,value},runtimeConfig{element{id,value}},attachedContexts{id,contextName}}},pageInfo{endCursor,hasNextPage,hasPreviousPage}}}", + "variables": { + "input": { + "first": 10, + "after": null, + "fullTextSearch": null, + "orderBy": null, + "predicates": [ + {"field": "repository", "constraint": {"stringMatches": ["test/test"], "booleanEquals": null, "enumEquals": null}}, + {"field": "projectRoot", "constraint": {"stringMatches": ["nonexistent/path"], "booleanEquals": null, "enumEquals": null}} + ] + } + } + }` + assert.JSONEq(t, expectedJSON, string(s)) + + response := `{ + "data": { + "searchStacks": { + "edges": [], + "pageInfo": { + "endCursor": null, + "hasNextPage": false, + "hasPreviousPage": false + } + } + } + }` + _, err = w.Write([]byte(response)) assert.NoError(t, err) } }, }, { - name: "returns empty map if no stack is found", + name: "returns nil if ModulePath is missing", + fields: fields{ + Metadata: vcs.Metadata{ + Remote: vcs.Remote{ + Name: "test/test", + }, + }, + }, + args: args{ + options: RemoteVarLoaderOptions{ + ModulePath: "", + }, + }, + want: nil, + wantErr: assert.NoError, + testServerFunc: func(t *testing.T) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // This test should not reach the server as ModulePath is empty + assert.FailNow(t, "HTTP server should not be called when ModulePath is empty") + } + }, + }, + { + name: "warns if ModulePath matches but no environment matches", fields: fields{ Metadata: vcs.Metadata{ Remote: vcs.Remote{ @@ -105,26 +385,75 @@ func TestSpaceliftRemoteVariableLoader_Load(t *testing.T) { }, args: args{ options: RemoteVarLoaderOptions{ - EnvName: "dev", + ModulePath: "module/path", + Environment: "nonexistent-env", }, }, want: nil, wantErr: assert.NoError, testServerFunc: func(t *testing.T) http.HandlerFunc { - // test that the /graqphl endpoint has been called and the correct query has been sent return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") assert.Equal(t, "POST", r.Method) s, err := io.ReadAll(r.Body) assert.NoError(t, err) - assert.JSONEq(t, `{"query":"query($input:SearchInput!){searchStacks(input: $input){edges{node{id,name,config{id,value}}},pageInfo{endCursor,hasNextPage,hasPreviousPage}}}","variables":{"input":{"first":2,"after":null,"fullTextSearch":"dev","predicates":[{"field":"repository","constraint":{"booleanEquals":null,"enumEquals":null,"stringMatches":["test/test"]}}],"orderBy":null}}}`, string(s)) - _, err = w.Write([]byte(`{"data":{"searchStacks":{"edges":[],"pageInfo":{"endCursor":"MDFKNEtQMzVETjFYRDcxNTY1UkJCMzBITUQ=","hasNextPage":true,"hasPreviousPage":false}}}}`)) + expectedJSON := `{ + "query": "query($input:SearchInput!){searchStacks(input: $input){edges{node{id,name,projectRoot,config{id,value},runtimeConfig{element{id,value}},attachedContexts{id,contextName}}},pageInfo{endCursor,hasNextPage,hasPreviousPage}}}", + "variables": { + "input": { + "first": 10, + "after": null, + "fullTextSearch": null, + "orderBy": null, + "predicates": [ + {"field": "repository", "constraint": {"stringMatches": ["test/test"], "booleanEquals": null, "enumEquals": null}}, + {"field": "projectRoot", "constraint": {"stringMatches": ["module/path"], "booleanEquals": null, "enumEquals": null}} + ] + } + } + }` + assert.JSONEq(t, expectedJSON, string(s)) + + response := `{ + "data": { + "searchStacks": { + "edges": [ + { + "node": { + "id": "module-path-dev", + "name": "module-path:dev", + "projectRoot": "module/path", + "attachedContexts": [ + {"id": "TF_VAR_context_var", "contextName": "context_value"} + ], + "config": [ + {"id": "TF_VAR_config_var", "value": "config_value"} + ], + "runtimeConfig": [ + {"element": {"id": "TF_VAR_runtime_var", "value": "runtime_value"}} + ] + } + } + ], + "pageInfo": { + "endCursor": null, + "hasNextPage": false, + "hasPreviousPage": false + } + } + } + }` + _, err = w.Write([]byte(response)) assert.NoError(t, err) } }, + verifyLog: func(t *testing.T, logOutput string) { + assert.Contains(t, logOutput, "no Spacelift stack found for module path") + }, }, { - name: "returns empty map if more than one stack is found", + name: "warns if multiple stacks are found and no environment is specified", fields: fields{ Metadata: vcs.Metadata{ Remote: vcs.Remote{ @@ -134,23 +463,88 @@ func TestSpaceliftRemoteVariableLoader_Load(t *testing.T) { }, args: args{ options: RemoteVarLoaderOptions{ - EnvName: "dev", + ModulePath: "module/path", + Environment: "", }, }, want: nil, wantErr: assert.NoError, testServerFunc: func(t *testing.T) http.HandlerFunc { - // test that the /graqphl endpoint has been called and the correct query has been sent return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") assert.Equal(t, "POST", r.Method) s, err := io.ReadAll(r.Body) assert.NoError(t, err) - assert.JSONEq(t, `{"query":"query($input:SearchInput!){searchStacks(input: $input){edges{node{id,name,config{id,value}}},pageInfo{endCursor,hasNextPage,hasPreviousPage}}}","variables":{"input":{"first":2,"after":null,"fullTextSearch":"dev","predicates":[{"field":"repository","constraint":{"booleanEquals":null,"enumEquals":null,"stringMatches":["test/test"]}}],"orderBy":null}}}`, string(s)) - _, err = w.Write([]byte(`{"data":{"searchStacks":{"edges":[{"node":{"id":"dev-a","name":"dev-a","config":[]}},{"node":{"id":"dev-b","name":"dev-b","config":[]}}],"pageInfo":{"endCursor":"MDFKNEtQMzVETjFYRDcxNTY1UkJCMzBITUQ=","hasNextPage":true,"hasPreviousPage":false}}}}`)) + expectedJSON := `{ + "query": "query($input:SearchInput!){searchStacks(input: $input){edges{node{id,name,projectRoot,config{id,value},runtimeConfig{element{id,value}},attachedContexts{id,contextName}}},pageInfo{endCursor,hasNextPage,hasPreviousPage}}}", + "variables": { + "input": { + "first": 10, + "after": null, + "fullTextSearch": null, + "orderBy": null, + "predicates": [ + {"field": "repository", "constraint": {"stringMatches": ["test/test"], "booleanEquals": null, "enumEquals": null}}, + {"field": "projectRoot", "constraint": {"stringMatches": ["module/path"], "booleanEquals": null, "enumEquals": null}} + ] + } + } + }` + assert.JSONEq(t, expectedJSON, string(s)) + + response := `{ + "data": { + "searchStacks": { + "edges": [ + { + "node": { + "id": "module-path-stack1", + "name": "module-path-stack1", + "projectRoot": "module/path", + "attachedContexts": [ + {"id": "TF_VAR_var1", "contextName": "context_value1"} + ], + "config": [ + {"id": "TF_VAR_var1", "value": "config_value1"} + ], + "runtimeConfig": [ + {"element": {"id": "TF_VAR_var1", "value": "runtime_value1"}} + ] + } + }, + { + "node": { + "id": "module-path-stack2", + "name": "module-path-stack2", + "projectRoot": "module/path", + "attachedContexts": [ + {"id": "TF_VAR_var2", "contextName": "context_value2"} + ], + "config": [ + {"id": "TF_VAR_var2", "value": "config_value2"} + ], + "runtimeConfig": [ + {"element": {"id": "TF_VAR_var2", "value": "runtime_value2"}} + ] + } + } + ], + "pageInfo": { + "endCursor": null, + "hasNextPage": false, + "hasPreviousPage": false + } + } + } + }` + _, err = w.Write([]byte(response)) assert.NoError(t, err) } }, + verifyLog: func(t *testing.T, logOutput string) { + assert.Contains(t, logOutput, "found multiple Spacelift stacks for module path") + }, }, } for _, tt := range tests { @@ -158,6 +552,13 @@ func TestSpaceliftRemoteVariableLoader_Load(t *testing.T) { ts := httptest.NewServer(tt.testServerFunc(t)) defer ts.Close() + var logBuf bytes.Buffer + log := zerolog.New(&logBuf).With().Timestamp().Logger() + + oldLogger := logging.Logger + logging.Logger = log + defer func() { logging.Logger = oldLogger }() + s := &SpaceliftRemoteVariableLoader{ Client: client.New(http.DefaultClient, &stubSession{ token: "test", @@ -172,6 +573,10 @@ func TestSpaceliftRemoteVariableLoader_Load(t *testing.T) { return } assert.Equalf(t, tt.want, got, "Load(%v)", tt.args.options) + + if tt.verifyLog != nil { + tt.verifyLog(t, logBuf.String()) + } }) } } diff --git a/internal/providers/terraform/hcl_provider.go b/internal/providers/terraform/hcl_provider.go index 4bac2ad8fdf..3899077c5fb 100644 --- a/internal/providers/terraform/hcl_provider.go +++ b/internal/providers/terraform/hcl_provider.go @@ -174,9 +174,15 @@ func NewHCLProvider(ctx *config.ProjectContext, rootPath hcl.RootPath, config *H options = append(options, hcl.OptionGraphEvaluator()) } + if ctx.ProjectConfig.Name != "" { + options = append(options, hcl.OptionWithProjectName(ctx.ProjectConfig.Name)) + } + + envMatcher := hcl.CreateEnvFileMatcher(ctx.RunContext.Config.Autodetect.EnvNames, ctx.RunContext.Config.Autodetect.TerraformVarFileExtensions) + return &HCLProvider{ policyClient: policyClient, - Parser: hcl.NewParser(rootPath, hcl.CreateEnvFileMatcher(ctx.RunContext.Config.Autodetect.EnvNames, ctx.RunContext.Config.Autodetect.TerraformVarFileExtensions), loader, logger, options...), + Parser: hcl.NewParser(rootPath, envMatcher, loader, logger, options...), planJSONParser: NewParser(ctx, true), ctx: ctx, config: *config, @@ -186,14 +192,6 @@ func NewHCLProvider(ctx *config.ProjectContext, rootPath hcl.RootPath, config *H func (p *HCLProvider) Context() *config.ProjectContext { return p.ctx } func (p *HCLProvider) ProjectName() string { - if p.ctx.ProjectConfig.Name != "" { - return p.ctx.ProjectConfig.Name - } - - if p.ctx.ProjectConfig.TerraformWorkspace != "" { - return p.Parser.ProjectName() + "-" + p.ctx.ProjectConfig.TerraformWorkspace - } - return p.Parser.ProjectName() } From 54ec148e9cb41c7495fae857848ae6bc87ce7034 Mon Sep 17 00:00:00 2001 From: Owen Date: Fri, 8 Nov 2024 15:09:51 +0100 Subject: [PATCH 34/81] chore: add an env var for json dump path (#3240) Add a new env var `INFRACOST_JSON_DUMP_PATH` to specify a target path to drop the plan files when the `INFRACOST_JSON_DUMP` env var has been specified. This is to support policy engine processing the plan file --- internal/providers/terraform/hcl_provider.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/providers/terraform/hcl_provider.go b/internal/providers/terraform/hcl_provider.go index 3899077c5fb..5e1f2069d09 100644 --- a/internal/providers/terraform/hcl_provider.go +++ b/internal/providers/terraform/hcl_provider.go @@ -331,7 +331,11 @@ func (p *HCLProvider) LoadPlanJSON() HCLProject { module.JSON, module.Error = p.modulesToPlanJSON(module.Module) if os.Getenv("INFRACOST_JSON_DUMP") == "true" { - err := os.WriteFile(fmt.Sprintf("%s-out.json", strings.ReplaceAll(module.Module.ModulePath, "/", "-")), module.JSON, os.ModePerm) // nolint: gosec + targetPath := fmt.Sprintf("%s-out.json", strings.ReplaceAll(module.Module.ModulePath, "/", "-")) + if os.Getenv("INFRACOST_JSON_DUMP_PATH") != "" { + targetPath = filepath.Join(os.Getenv("INFRACOST_JSON_DUMP_PATH"), targetPath) + } + err := os.WriteFile(targetPath, module.JSON, os.ModePerm) // nolint: gosec if err != nil { p.logger.Debug().Err(err).Msg("failed to write to json dump") } From d8076a8e7a2ef26d6bf287b9b19c592921a74e40 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Tue, 12 Nov 2024 11:56:04 +0000 Subject: [PATCH 35/81] fix(google): `google_compute_instance_group_manager` with multiple disks Before this was messing up the diffs since the cost components had the same name. I've updated this to use subresources, similar to how the AWS instances work so it avoids this. --- .../google/compute_instance_group_manager.go | 12 ++++-- .../compute_region_instance_group_manager.go | 15 ++++--- ...compute_instance_group_manager_test.golden | 23 +++++++---- .../compute_instance_group_manager_test.tf | 39 ++++++++++++++++++ .../compute_per_instance_config_test.golden | 3 +- ..._region_instance_group_manager_test.golden | 23 +++++++---- ...pute_region_instance_group_manager_test.tf | 40 +++++++++++++++++++ ...ute_region_per_instance_config_test.golden | 3 +- internal/resources/google/compute_disk.go | 16 +++++--- .../google/compute_instance_group_manager.go | 5 ++- .../compute_region_instance_group_manager.go | 5 ++- 11 files changed, 151 insertions(+), 33 deletions(-) diff --git a/internal/providers/terraform/google/compute_instance_group_manager.go b/internal/providers/terraform/google/compute_instance_group_manager.go index 50ba6372fbd..ceeab57de86 100644 --- a/internal/providers/terraform/google/compute_instance_group_manager.go +++ b/internal/providers/terraform/google/compute_instance_group_manager.go @@ -1,6 +1,8 @@ package google import ( + "fmt" + "github.com/infracost/infracost/internal/resources/google" "github.com/infracost/infracost/internal/schema" ) @@ -49,7 +51,7 @@ func newComputeInstanceGroupManager(d *schema.ResourceData) schema.CoreResource purchaseOption = getComputePurchaseOption(instanceTemplate.RawValues) if len(instanceTemplate.Get("disk").Array()) > 0 { - for _, disk := range instanceTemplate.Get("disk").Array() { + for i, disk := range instanceTemplate.Get("disk").Array() { diskType := disk.Get("type").String() switch diskType { @@ -63,11 +65,13 @@ func newComputeInstanceGroupManager(d *schema.ResourceData) schema.CoreResource diskType := disk.Get("disk_type").String() disks = append(disks, &google.ComputeDisk{ - Type: diskType, - Size: float64(diskSize), + Address: fmt.Sprintf("disk[%d]", i), + Type: diskType, + Size: float64(diskSize), + Region: d.Region, + InstanceCount: &targetSize, }) } - } } diff --git a/internal/providers/terraform/google/compute_region_instance_group_manager.go b/internal/providers/terraform/google/compute_region_instance_group_manager.go index ef8d2a2cd6d..5bcbb38d441 100644 --- a/internal/providers/terraform/google/compute_region_instance_group_manager.go +++ b/internal/providers/terraform/google/compute_region_instance_group_manager.go @@ -1,6 +1,8 @@ package google import ( + "fmt" + "github.com/infracost/infracost/internal/resources/google" "github.com/infracost/infracost/internal/schema" ) @@ -18,8 +20,6 @@ func getComputeRegionInstanceGroupManagerRegistryItem() *schema.RegistryItem { } func newComputeRegionInstanceGroupManager(d *schema.ResourceData) schema.CoreResource { - region := d.Get("region").String() - targetSize := int64(1) if d.Get("target_size").Exists() { targetSize = d.Get("target_size").Int() @@ -43,7 +43,7 @@ func newComputeRegionInstanceGroupManager(d *schema.ResourceData) schema.CoreRes purchaseOption = "preemptible" } - for _, disk := range instanceTemplate.Get("disk").Array() { + for i, disk := range instanceTemplate.Get("disk").Array() { diskType := disk.Get("type").String() switch diskType { case "SCRATCH": @@ -56,8 +56,11 @@ func newComputeRegionInstanceGroupManager(d *schema.ResourceData) schema.CoreRes diskType := disk.Get("disk_type").String() disks = append(disks, &google.ComputeDisk{ - Type: diskType, - Size: float64(diskSize), + Address: fmt.Sprintf("disk[%d]", i), + Type: diskType, + Size: float64(diskSize), + Region: d.Region, + InstanceCount: &targetSize, }) } } @@ -67,7 +70,7 @@ func newComputeRegionInstanceGroupManager(d *schema.ResourceData) schema.CoreRes r := &google.ComputeRegionInstanceGroupManager{ Address: d.Address, - Region: region, + Region: d.Region, MachineType: machineType, PurchaseOption: purchaseOption, TargetSize: targetSize, diff --git a/internal/providers/terraform/google/testdata/compute_instance_group_manager_test/compute_instance_group_manager_test.golden b/internal/providers/terraform/google/testdata/compute_instance_group_manager_test/compute_instance_group_manager_test.golden index 001ada6c21d..98b89924a72 100644 --- a/internal/providers/terraform/google/testdata/compute_instance_group_manager_test/compute_instance_group_manager_test.golden +++ b/internal/providers/terraform/google/testdata/compute_instance_group_manager_test/compute_instance_group_manager_test.golden @@ -3,21 +3,30 @@ google_compute_instance_group_manager.default ├─ Instance usage (Linux/UNIX, on-demand, f1-micro) 2,920 hours $15.53 - ├─ SSD provisioned storage (pd-ssd) 1,500 GB $255.00 ├─ Local SSD provisioned storage 1,500 GB $120.00 - └─ NVIDIA Tesla K80 (on-demand) 5,840 hours $1,839.60 + ├─ NVIDIA Tesla K80 (on-demand) 5,840 hours $1,839.60 + └─ disk[0] + └─ SSD provisioned storage (pd-ssd) 1,500 GB $255.00 - OVERALL TOTAL $2,230.13 + google_compute_instance_group_manager.two_disks + ├─ Instance usage (Linux/UNIX, on-demand, f1-micro) 2,920 hours $15.53 + ├─ NVIDIA Tesla K80 (on-demand) 5,840 hours $1,839.60 + ├─ disk[0] + │ └─ SSD provisioned storage (pd-ssd) 40 GB $6.80 + └─ disk[1] + └─ SSD provisioned storage (pd-ssd) 200 GB $34.00 + + OVERALL TOTAL $4,126.07 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. ────────────────────────────────── -2 cloud resources were detected: -∙ 1 was estimated -∙ 1 was free +4 cloud resources were detected: +∙ 2 were estimated +∙ 2 were free ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $2,230 ┃ - ┃ $2,230 ┃ +┃ main ┃ $4,126 ┃ - ┃ $4,126 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/google/testdata/compute_instance_group_manager_test/compute_instance_group_manager_test.tf b/internal/providers/terraform/google/testdata/compute_instance_group_manager_test/compute_instance_group_manager_test.tf index 18a68863ab5..4b8f332100d 100644 --- a/internal/providers/terraform/google/testdata/compute_instance_group_manager_test/compute_instance_group_manager_test.tf +++ b/internal/providers/terraform/google/testdata/compute_instance_group_manager_test/compute_instance_group_manager_test.tf @@ -46,3 +46,42 @@ resource "google_compute_instance_group_manager" "default" { target_size = 4 } + +resource "google_compute_instance_template" "two_disks" { + name = "two-disks-template" + + machine_type = "f1-micro" + + scheduling { + automatic_restart = true + on_host_maintenance = "MIGRATE" + } + + disk { + disk_type = "pd-ssd" + disk_size_gb = "10" + } + + disk { + disk_type = "pd-ssd" + disk_size_gb = "50" + } + + guest_accelerator { + type = "nvidia-tesla-k80" + count = 2 + } +} + +resource "google_compute_instance_group_manager" "two_disks" { + name = "two_disks" + + base_instance_name = "app" + zone = "us-central1-a" + + version { + instance_template = google_compute_instance_template.two_disks.id + } + + target_size = 4 +} diff --git a/internal/providers/terraform/google/testdata/compute_per_instance_config_test/compute_per_instance_config_test.golden b/internal/providers/terraform/google/testdata/compute_per_instance_config_test/compute_per_instance_config_test.golden index 40be266825a..a288b49ed9d 100644 --- a/internal/providers/terraform/google/testdata/compute_per_instance_config_test/compute_per_instance_config_test.golden +++ b/internal/providers/terraform/google/testdata/compute_per_instance_config_test/compute_per_instance_config_test.golden @@ -3,7 +3,8 @@ google_compute_instance_group_manager.default ├─ Instance usage (Linux/UNIX, on-demand, f1-micro) 730 hours $3.88 - └─ SSD provisioned storage (pd-ssd) 375 GB $63.75 + └─ disk[0] + └─ SSD provisioned storage (pd-ssd) 375 GB $63.75 OVERALL TOTAL $67.63 diff --git a/internal/providers/terraform/google/testdata/compute_region_instance_group_manager_test/compute_region_instance_group_manager_test.golden b/internal/providers/terraform/google/testdata/compute_region_instance_group_manager_test/compute_region_instance_group_manager_test.golden index 41cd95e98c8..61ffcba726c 100644 --- a/internal/providers/terraform/google/testdata/compute_region_instance_group_manager_test/compute_region_instance_group_manager_test.golden +++ b/internal/providers/terraform/google/testdata/compute_region_instance_group_manager_test/compute_region_instance_group_manager_test.golden @@ -1,22 +1,31 @@ Name Monthly Qty Unit Monthly Cost + google_compute_region_instance_group_manager.two_disks + ├─ Instance usage (Linux/UNIX, on-demand, f1-micro) 2,920 hours $15.53 + ├─ NVIDIA Tesla K80 (on-demand) 5,840 hours $1,839.60 + ├─ disk[0] + │ └─ SSD provisioned storage (pd-ssd) 40 GB $6.80 + └─ disk[1] + └─ SSD provisioned storage (pd-ssd) 200 GB $34.00 + google_compute_region_instance_group_manager.appserver ├─ Instance usage (Linux/UNIX, on-demand, n1-standard-16) 2,190 hours $1,165.07 - ├─ Balanced provisioned storage (pd-balanced) 1,200 GB $120.00 - └─ Local SSD provisioned storage 1,125 GB $90.00 + ├─ Local SSD provisioned storage 1,125 GB $90.00 + └─ disk[0] + └─ Balanced provisioned storage (pd-balanced) 1,200 GB $120.00 - OVERALL TOTAL $1,375.07 + OVERALL TOTAL $3,271.01 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. ────────────────────────────────── -2 cloud resources were detected: -∙ 1 was estimated -∙ 1 was free +4 cloud resources were detected: +∙ 2 were estimated +∙ 2 were free ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $1,375 ┃ - ┃ $1,375 ┃ +┃ main ┃ $3,271 ┃ - ┃ $3,271 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/google/testdata/compute_region_instance_group_manager_test/compute_region_instance_group_manager_test.tf b/internal/providers/terraform/google/testdata/compute_region_instance_group_manager_test/compute_region_instance_group_manager_test.tf index 9da678f970a..8743daf8648 100644 --- a/internal/providers/terraform/google/testdata/compute_region_instance_group_manager_test/compute_region_instance_group_manager_test.tf +++ b/internal/providers/terraform/google/testdata/compute_region_instance_group_manager_test/compute_region_instance_group_manager_test.tf @@ -41,3 +41,43 @@ resource "google_compute_region_instance_group_manager" "appserver" { target_size = 3 } + + +resource "google_compute_instance_template" "two_disks" { + name = "two-disks-template" + + machine_type = "f1-micro" + + scheduling { + automatic_restart = true + on_host_maintenance = "MIGRATE" + } + + disk { + disk_type = "pd-ssd" + disk_size_gb = "10" + } + + disk { + disk_type = "pd-ssd" + disk_size_gb = "50" + } + + guest_accelerator { + type = "nvidia-tesla-k80" + count = 2 + } +} + +resource "google_compute_region_instance_group_manager" "two_disks" { + name = "two_disks" + + base_instance_name = "app" + region = "us-central1" + + version { + instance_template = google_compute_instance_template.two_disks.id + } + + target_size = 4 +} diff --git a/internal/providers/terraform/google/testdata/compute_region_per_instance_config_test/compute_region_per_instance_config_test.golden b/internal/providers/terraform/google/testdata/compute_region_per_instance_config_test/compute_region_per_instance_config_test.golden index 2da39f90637..9070425fbc9 100644 --- a/internal/providers/terraform/google/testdata/compute_region_per_instance_config_test/compute_region_per_instance_config_test.golden +++ b/internal/providers/terraform/google/testdata/compute_region_per_instance_config_test/compute_region_per_instance_config_test.golden @@ -3,7 +3,8 @@ google_compute_region_instance_group_manager.appserver ├─ Instance usage (Linux/UNIX, on-demand, f1-micro) 730 hours $3.88 - └─ SSD provisioned storage (pd-ssd) 375 GB $63.75 + └─ disk[0] + └─ SSD provisioned storage (pd-ssd) 375 GB $63.75 OVERALL TOTAL $67.63 diff --git a/internal/resources/google/compute_disk.go b/internal/resources/google/compute_disk.go index f49c951dcf4..3dfca154e02 100644 --- a/internal/resources/google/compute_disk.go +++ b/internal/resources/google/compute_disk.go @@ -7,10 +7,11 @@ import ( // ComputeDisk struct represents Compute Disk resource. type ComputeDisk struct { - Address string - Region string - Type string - Size float64 + Address string + Region string + Type string + Size float64 + InstanceCount *int64 // applicable for pd-extreme and hyperdisk-extreme IOPS int64 @@ -35,8 +36,13 @@ func (r *ComputeDisk) PopulateUsage(u *schema.UsageData) { // This method is called after the resource is initialised by an IaC provider. // See providers folder for more information. func (r *ComputeDisk) BuildResource() *schema.Resource { + count := int64(1) + if r.InstanceCount != nil { + count = *r.InstanceCount + } + costComponents := []*schema.CostComponent{ - computeDiskCostComponent(r.Region, r.Type, r.Size, 1), + computeDiskCostComponent(r.Region, r.Type, r.Size, count), } if r.Type == "pd-extreme" || r.Type == "hyperdisk-extreme" { diff --git a/internal/resources/google/compute_instance_group_manager.go b/internal/resources/google/compute_instance_group_manager.go index 196185db3c0..80a783bcfc1 100644 --- a/internal/resources/google/compute_instance_group_manager.go +++ b/internal/resources/google/compute_instance_group_manager.go @@ -45,8 +45,10 @@ func (r *ComputeInstanceGroupManager) BuildResource() *schema.Resource { return nil } + subResources := make([]*schema.Resource, 0) + for _, disk := range r.Disks { - costComponents = append(costComponents, computeDiskCostComponent(r.Region, disk.Type, disk.Size, r.TargetSize)) + subResources = append(subResources, disk.BuildResource()) } if r.ScratchDisks > 0 { @@ -62,6 +64,7 @@ func (r *ComputeInstanceGroupManager) BuildResource() *schema.Resource { return &schema.Resource{ Name: r.Address, UsageSchema: r.UsageSchema(), + SubResources: subResources, CostComponents: costComponents, } } diff --git a/internal/resources/google/compute_region_instance_group_manager.go b/internal/resources/google/compute_region_instance_group_manager.go index b080ba7285b..e1e4b161e42 100644 --- a/internal/resources/google/compute_region_instance_group_manager.go +++ b/internal/resources/google/compute_region_instance_group_manager.go @@ -45,8 +45,10 @@ func (r *ComputeRegionInstanceGroupManager) BuildResource() *schema.Resource { return nil } + subResources := make([]*schema.Resource, 0) + for _, disk := range r.Disks { - costComponents = append(costComponents, computeDiskCostComponent(r.Region, disk.Type, disk.Size, r.TargetSize)) + subResources = append(subResources, disk.BuildResource()) } if r.ScratchDisks > 0 { @@ -62,6 +64,7 @@ func (r *ComputeRegionInstanceGroupManager) BuildResource() *schema.Resource { return &schema.Resource{ Name: r.Address, UsageSchema: r.UsageSchema(), + SubResources: subResources, CostComponents: costComponents, } } From 8fbbe21293317afde342262553aec0ac1c93e70a Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Mon, 11 Nov 2024 18:51:46 +0000 Subject: [PATCH 36/81] feat: add a remote S3 cache to the module loader --- go.mod | 12 +- go.sum | 21 ++ internal/config/config.go | 4 + internal/hcl/modules/fetch.go | 122 ++++-- internal/hcl/modules/fetch_test.go | 171 +++++++++ internal/hcl/modules/loader.go | 36 +- internal/hcl/modules/loader_test.go | 21 +- internal/hcl/modules/s3_cache.go | 152 ++++++++ internal/hcl/parser_test.go | 351 ++++++++++++++++-- internal/providers/terraform/hcl_provider.go | 20 +- .../providers/terraform/hcl_provider_test.go | 9 +- 11 files changed, 830 insertions(+), 89 deletions(-) create mode 100644 internal/hcl/modules/s3_cache.go diff --git a/go.mod b/go.mod index 71c7c0fa4b0..63abf281700 100644 --- a/go.mod +++ b/go.mod @@ -107,6 +107,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.6 github.com/hashicorp/terraform-svchost v0.1.1 github.com/maruel/panicparse/v2 v2.3.1 + github.com/mholt/archiver/v3 v3.5.1 github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c github.com/pkg/profile v1.2.1 @@ -118,6 +119,7 @@ require ( github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7 github.com/withfig/autocomplete-tools/packages/cobra v1.2.0 github.com/xanzy/go-gitlab v0.86.0 + golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/oauth2 v0.21.0 k8s.io/apimachinery v0.29.2 ) @@ -142,6 +144,7 @@ require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect github.com/agnivade/levenshtein v1.1.1 // indirect + github.com/andybalholm/brotli v1.0.1 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/apparentlymart/go-versions v1.0.1 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.25 // indirect @@ -156,12 +159,14 @@ require ( github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dlclark/regexp2 v1.8.1 // indirect + github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-jose/go-jose/v4 v4.0.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/google/go-github/v35 v35.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect github.com/goware/prefixer v0.0.0-20160118172347-395022866408 // indirect @@ -186,12 +191,15 @@ require ( github.com/jessevdk/go-flags v1.5.0 // indirect github.com/jstemmer/go-junit-report v1.0.0 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/klauspost/pgzip v1.2.5 // indirect github.com/lib/pq v1.10.5 // indirect github.com/mattn/go-zglob v0.0.3 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/panicwrap v1.0.0 // indirect + github.com/nwaples/rardecode v1.1.0 // indirect github.com/oklog/run v1.1.0 // indirect github.com/owenrumney/go-sarif v1.1.1 // indirect + github.com/pierrec/lz4/v4 v4.1.2 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect @@ -209,10 +217,10 @@ require ( github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect + github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect go.mozilla.org/gopgagent v0.0.0-20170926210634-4d7ea76ff71a // indirect go.mozilla.org/sops/v3 v3.7.3 // indirect - golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/term v0.21.0 // indirect golang.org/x/time v0.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect @@ -227,7 +235,7 @@ require ( github.com/OneOfOne/xxhash v1.2.8 // indirect github.com/apparentlymart/go-cidr v1.1.0 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect - github.com/aws/aws-sdk-go v1.44.122 // indirect + github.com/aws/aws-sdk-go v1.44.122 github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/bmatcuk/doublestar v1.3.4 github.com/ghodss/yaml v1.0.0 // indirect diff --git a/go.sum b/go.sum index 4e2112210b9..09c6b51d5ef 100644 --- a/go.sum +++ b/go.sum @@ -284,6 +284,8 @@ github.com/aliscott/go-pretty/v6 v6.1.1-0.20210226104003-408905a61c8e/go.mod h1: github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190329064014-6e358769c32a/go.mod h1:T9M45xf79ahXVelWoOBmH0y4aC1t5kXO5BxwyakgIGA= github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190103054945-8205d1f41e70/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/aliyun/aliyun-tablestore-go-sdk v4.1.2+incompatible/go.mod h1:LDQHRZylxvcg8H7wBIDfvO5g/cy4/sz1iucBlc2l3Jw= +github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc= +github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antchfx/xpath v0.0.0-20190129040759-c8489ed3251e/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= @@ -468,6 +470,9 @@ github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5Xh github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= +github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dylanmei/iso8601 v0.1.0/go.mod h1:w9KhXSgIyROl1DefbMYIE7UVSIvELTbMrCfx+QkYnoQ= @@ -600,6 +605,7 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -852,8 +858,13 @@ github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= +github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -913,6 +924,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= +github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= github.com/miekg/dns v1.0.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= @@ -960,6 +973,8 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U= +github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= +github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= @@ -995,6 +1010,8 @@ github.com/owenrumney/go-sarif v1.1.1 h1:QNObu6YX1igyFKhdzd7vgzmw7XsWN3/6NMGuDzB github.com/owenrumney/go-sarif v1.1.1/go.mod h1:dNDiPlF04ESR/6fHlPyq7gHKmrM0sHUvAGjsoh8ZH0U= github.com/packer-community/winrmcp v0.0.0-20180921211025-c76d91c1e7db/go.mod h1:f6Izs6JvFTdnRbziASagjZ2vmf55NSIkC/weStxCHqk= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pierrec/lz4/v4 v4.1.2 h1:qvY3YFXRQE/XB8MlLzJH7mSzBs74eA2gg52YTk6jUPM= +github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/browser v0.0.0-20201207095918-0426ae3fba23/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= @@ -1131,6 +1148,8 @@ github.com/tombuildsstuff/giovanni v0.15.1/go.mod h1:0TZugJPEtqzPlMpuJHYfXY6Dq2u github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7 h1:qDMxFVd8Zo0rIhnEBdCIbR+T6WgjwkxpFZMN8zZmmjg= github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7/go.mod h1:5hzpfalEjfcJWp9yq75/EZoEu2Mzm34eJAPm3HOW2tw= github.com/ugorji/go v0.0.0-20180813092308-00b869d2f4a5/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= +github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= @@ -1157,6 +1176,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20161029104018-1d6e34225557/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw= diff --git a/internal/config/config.go b/internal/config/config.go index a572b63661f..1f018dd62f0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -159,6 +159,10 @@ type Config struct { // TerraformSourceMap replaces any source URL with the provided value. TerraformSourceMap TerraformSourceMap `envconfig:"TERRAFORM_SOURCE_MAP"` + S3ModuleCacheRegion string `envconfig:"S3_MODULE_CACHE_REGION"` + S3ModuleCacheBucket string `envconfig:"S3_MODULE_CACHE_BUCKET"` + S3ModuleCachePrefix string `envconfig:"S3_MODULE_CACHE_PREFIX"` + // Org settings EnableCloudForOrganization bool diff --git a/internal/hcl/modules/fetch.go b/internal/hcl/modules/fetch.go index f4daae93a0b..ab43cf49a3d 100644 --- a/internal/hcl/modules/fetch.go +++ b/internal/hcl/modules/fetch.go @@ -18,57 +18,115 @@ import ( // PackageFetcher downloads modules from a remote source to the given destination // This supports all the non-local and non-Terraform registry sources listed here: https://www.terraform.io/language/modules/sources type PackageFetcher struct { - cache sync.Map - logger zerolog.Logger + localCache sync.Map + remoteCache RemoteCache + logger zerolog.Logger } // NewPackageFetcher constructs a new package fetcher -func NewPackageFetcher(logger zerolog.Logger) *PackageFetcher { +func NewPackageFetcher(remoteCache RemoteCache, logger zerolog.Logger) *PackageFetcher { return &PackageFetcher{ - logger: logger, + remoteCache: remoteCache, + logger: logger, } } // fetch downloads the remote module using the go-getter library // See: https://github.com/hashicorp/go-getter -func (r *PackageFetcher) fetch(moduleAddr string, dest string) error { - if v, ok := r.cache.Load(moduleAddr); ok { - prevDest, _ := v.(string) +func (p *PackageFetcher) fetch(moduleAddr string, dest string) error { + fetched, err := p.fetchFromLocalCache(moduleAddr, dest) + if fetched { + p.logger.Debug().Msgf("fetched module %s from local cache", moduleAddr) + return nil + } - r.logger.Debug().Msgf("module %s already downloaded, copying from '%s' to '%s'", moduleAddr, prevDest, dest) + if err != nil { + p.logger.Warn().Msgf("error fetching module %s from local cache, trying to fetch from elsewhere: %s", moduleAddr, err) + } - err := os.Mkdir(dest, os.ModePerm) - if err != nil { - return fmt.Errorf("failed to create directory '%s': %w", dest, err) + if p.remoteCache != nil { + fetched, err = p.fetchFromRemoteCache(moduleAddr, dest) + if fetched { + p.logger.Debug().Msgf("fetched module %s from remote cache", moduleAddr) + p.localCache.Store(moduleAddr, dest) + return nil } - // Skip dotfiles and create new symlinks to be consistent with what Terraform init does - opt := copy.Options{ - Skip: func(src string) (bool, error) { - return strings.HasPrefix(filepath.Base(src), "."), nil - }, - OnSymlink: func(src string) copy.SymlinkAction { - return copy.Shallow - }, + if err != nil { + p.logger.Warn().Msgf("error fetching module %s from remote cache, trying to fetch from elsewhere: %s", moduleAddr, err) } + } + + _, err = p.fetchFromRemote(moduleAddr, dest) + if err != nil { + return fmt.Errorf("error fetching module %s from remote: %w", moduleAddr, err) + } - err = copy.Copy(prevDest, dest, opt) + p.localCache.Store(moduleAddr, dest) + + if p.remoteCache != nil { + p.logger.Debug().Msgf("putting module %s into remote cache", moduleAddr) + err = p.remoteCache.Put(moduleAddr, dest) if err != nil { - return fmt.Errorf("failed to copy module from '%s' to '%s': %w", prevDest, dest, err) + p.logger.Warn().Msgf("error putting module %s into remote cache: %s", moduleAddr, err) } + } - return nil + return nil +} + +func (p *PackageFetcher) fetchFromLocalCache(moduleAddr, dest string) (bool, error) { + v, ok := p.localCache.Load(moduleAddr) + if !ok { + return false, nil } - var cached []string - r.cache.Range(func(k, value any) bool { - s, _ := value.(string) - cached = append(cached, s) - return true - }) + prevDest, _ := v.(string) + + p.logger.Debug().Msgf("module %s already downloaded, copying from '%s' to '%s'", moduleAddr, prevDest, dest) - r.logger.Debug().Strs("cached_module_addresses", cached).Msgf("module %s does not exist in cache, proceeding to download", moduleAddr) + err := os.Mkdir(dest, os.ModePerm) + if err != nil { + return false, fmt.Errorf("failed to create directory '%s': %w", dest, err) + } + // Skip dotfiles and create new symlinks to be consistent with what Terraform init does + opt := copy.Options{ + Skip: func(src string) (bool, error) { + return strings.HasPrefix(filepath.Base(src), "."), nil + }, + OnSymlink: func(src string) copy.SymlinkAction { + return copy.Shallow + }, + } + + err = copy.Copy(prevDest, dest, opt) + if err != nil { + return false, fmt.Errorf("failed to copy module from '%s' to '%s': %w", prevDest, dest, err) + } + + return true, nil +} + +func (p *PackageFetcher) fetchFromRemoteCache(moduleAddr, dest string) (bool, error) { + ok, err := p.remoteCache.Exists(moduleAddr) + if err != nil { + return false, err + } + + if !ok { + return false, nil + } + + err = p.remoteCache.Get(moduleAddr, dest) + if err != nil { + return false, err + } + + return true, nil +} + +func (p *PackageFetcher) fetchFromRemote(moduleAddr, dest string) (bool, error) { decompressors := map[string]getter.Decompressor{} for k, decompressor := range getter.Decompressors { decompressors[k] = decompressor @@ -94,14 +152,12 @@ func (r *PackageFetcher) fetch(moduleAddr string, dest string) error { Getters: getters, } - r.cache.Store(moduleAddr, dest) - err := client.Get() if err != nil { - return fmt.Errorf("could not download module %s to cache %w", moduleAddr, err) + return false, err } - return nil + return true, nil } // CustomGitGetter extends the standard GitGetter transforming SSH sources to diff --git a/internal/hcl/modules/fetch_test.go b/internal/hcl/modules/fetch_test.go index 6174ec71056..79345bba939 100644 --- a/internal/hcl/modules/fetch_test.go +++ b/internal/hcl/modules/fetch_test.go @@ -1,10 +1,18 @@ package modules import ( + "fmt" + "io" "net/url" + "os" + "path/filepath" + "sync" "testing" + getter "github.com/hashicorp/go-getter" + "github.com/rs/zerolog" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestTransformSSHToHTTPS(t *testing.T) { @@ -24,3 +32,166 @@ func TestTransformSSHToHTTPS(t *testing.T) { assert.Equal(t, tc.expected, transformed.String()) } } + +func TestPackageFetcher_fetch_RemoteCache(t *testing.T) { + if testing.Short() { + t.Skip("Skipping test in short mode") + } + + logger := zerolog.New(io.Discard) + mock := &mockRemoteCache{ + cache: make(map[string]string), + } + + tests := []struct { + name string + setupCache func(*mockRemoteCache, string) + expectedCalls map[string]int + expectedError bool + }{ + { + name: "should use cached module from remote cache", + setupCache: func(c *mockRemoteCache, tmpDir string) { + // Create a mock module directory with content + moduleDir := filepath.Join(tmpDir, "cached_module") + require.NoError(t, os.MkdirAll(moduleDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(moduleDir, "main.tf"), []byte(` + variable "instance_type" { + type = string + default = "t3.micro" + } + `), 0600)) + // Store the module directory path in the cache + require.NoError(t, c.Put("git::https://github.com/terraform-aws-modules/terraform-aws-vpc?ref=v5.15.0", moduleDir)) + }, + expectedCalls: map[string]int{ + "Exists": 1, + "Get": 1, + "Put": 0, + }, + }, + { + name: "should cache module to remote cache when not found", + setupCache: func(c *mockRemoteCache, tmpDir string) {}, + expectedCalls: map[string]int{ + "Exists": 1, + "Get": 0, + "Put": 1, + }, + }, + { + name: "should handle remote cache errors gracefully", + setupCache: func(c *mockRemoteCache, tmpDir string) { + c.shouldError = true + }, + expectedCalls: map[string]int{ + "Exists": 1, + "Get": 0, + "Put": 1, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + + mock.ResetCache() + tt.setupCache(mock, tmpDir) + mock.ResetCounters() + + // Create the test Terraform configuration + err := os.WriteFile(filepath.Join(tmpDir, "main.tf"), []byte(` + module "ec2_instance" { + source = "git::https://github.com/terraform-aws-modules/terraform-aws-vpc?ref=v5.15.0" + } + `), 0600) + require.NoError(t, err) + + fetcher := NewPackageFetcher(mock, logger) + + err = fetcher.fetch("git::https://github.com/terraform-aws-modules/terraform-aws-vpc?ref=v5.15.0", filepath.Join(tmpDir, "module")) + if tt.expectedError { + assert.Error(t, err) + return + } + assert.NoError(t, err) + + assert.Equal(t, tt.expectedCalls["Exists"], mock.existsCalls, "wrong number of Exists calls") + assert.Equal(t, tt.expectedCalls["Get"], mock.getCalls, "wrong number of Get calls") + assert.Equal(t, tt.expectedCalls["Put"], mock.putCalls, "wrong number of Put calls") + }) + } +} + +type mockRemoteCache struct { + cache map[string]string + existsCalls int + getCalls int + putCalls int + shouldError bool + mu sync.Mutex +} + +func (m *mockRemoteCache) Exists(key string) (bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + m.existsCalls++ + + if m.shouldError { + return false, fmt.Errorf("mock remote cache error") + } + + _, exists := m.cache[key] + return exists, nil +} + +func (m *mockRemoteCache) Get(key string, dest string) error { + m.mu.Lock() + defer m.mu.Unlock() + m.getCalls++ + + if m.shouldError { + return fmt.Errorf("mock remote cache error") + } + + srcDir, exists := m.cache[key] + if !exists { + return fmt.Errorf("key not found: %s", key) + } + + // Use go-getter to copy the directory + client := &getter.Client{ + Src: fmt.Sprintf("file://%s", srcDir), + Dst: dest, + Mode: getter.ClientModeDir, + } + + return client.Get() +} + +func (m *mockRemoteCache) Put(key string, src string) error { + m.mu.Lock() + defer m.mu.Unlock() + m.putCalls++ + + if m.shouldError { + return fmt.Errorf("mock remote cache error") + } + + m.cache[key] = src + return nil +} + +func (m *mockRemoteCache) ResetCache() { + m.mu.Lock() + defer m.mu.Unlock() + m.cache = make(map[string]string) +} + +func (m *mockRemoteCache) ResetCounters() { + m.existsCalls = 0 + m.getCalls = 0 + m.putCalls = 0 + m.shouldError = false +} diff --git a/internal/hcl/modules/loader.go b/internal/hcl/modules/loader.go index 9ce13eac461..7251db5f1c9 100644 --- a/internal/hcl/modules/loader.go +++ b/internal/hcl/modules/loader.go @@ -43,6 +43,13 @@ var ( maxSparseCheckoutDepth = 1 ) +// RemoteCache is an interface that defines the methods for a remote cache, i.e. an S3 bucket. +type RemoteCache interface { + Exists(key string) (bool, error) + Get(key string, dest string) error + Put(key string, src string) error +} + // ModuleLoader handles the loading of Terraform modules. It supports local, registry and other remote modules. // // The path should be the root directory of the Terraform project. We use a distinct module loader per Terraform project, @@ -69,24 +76,33 @@ type SourceMapResult struct { Version string RawQuery string } +type ModuleLoaderOptions struct { + CachePath string + HCLParser *SharedHCLParser + CredentialsSource *CredentialsSource + SourceMap config.TerraformSourceMap + Logger zerolog.Logger + ModuleSync *intSync.KeyMutex + RemoteCache RemoteCache +} // NewModuleLoader constructs a new module loader -func NewModuleLoader(cachePath string, hclParser *SharedHCLParser, credentialsSource *CredentialsSource, sourceMap config.TerraformSourceMap, logger zerolog.Logger, moduleSync *intSync.KeyMutex) *ModuleLoader { - fetcher := NewPackageFetcher(logger) +func NewModuleLoader(opts ModuleLoaderOptions) *ModuleLoader { + fetcher := NewPackageFetcher(opts.RemoteCache, opts.Logger) // we need to have a disco for each project that has defined credentials - d := NewDisco(credentialsSource, logger) + d := NewDisco(opts.CredentialsSource, opts.Logger) m := &ModuleLoader{ - cachePath: cachePath, - cache: NewCache(d, logger), - hclParser: hclParser, - sourceMap: sourceMap, + cachePath: opts.CachePath, + cache: NewCache(d, opts.Logger), + hclParser: opts.HCLParser, + sourceMap: opts.SourceMap, packageFetcher: fetcher, - logger: logger, - sync: moduleSync, + logger: opts.Logger, + sync: opts.ModuleSync, } - m.registryLoader = NewRegistryLoader(fetcher, d, logger) + m.registryLoader = NewRegistryLoader(fetcher, d, opts.Logger) return m } diff --git a/internal/hcl/modules/loader_test.go b/internal/hcl/modules/loader_test.go index ac2ea557dd5..287d27d4ead 100644 --- a/internal/hcl/modules/loader_test.go +++ b/internal/hcl/modules/loader_test.go @@ -8,9 +8,10 @@ import ( "sync" "testing" - "github.com/rs/zerolog" "github.com/stretchr/testify/assert" + "github.com/rs/zerolog" + "github.com/infracost/infracost/internal/config" "github.com/infracost/infracost/internal/credentials" sync2 "github.com/infracost/infracost/internal/sync" @@ -30,7 +31,14 @@ func testLoaderE2E(t *testing.T, path string, expectedModules []*ManifestModule, logger := zerolog.New(io.Discard) - moduleLoader := NewModuleLoader(path, NewSharedHCLParser(), &CredentialsSource{FetchToken: credentials.FindTerraformCloudToken}, opts.SourceMap, logger, &sync2.KeyMutex{}) + moduleLoader := NewModuleLoader(ModuleLoaderOptions{ + CachePath: path, + HCLParser: NewSharedHCLParser(), + CredentialsSource: &CredentialsSource{FetchToken: credentials.FindTerraformCloudToken}, + SourceMap: opts.SourceMap, + Logger: logger, + ModuleSync: &sync2.KeyMutex{}, + }) manifest, err := moduleLoader.Load(path) if !assert.NoError(t, err) { @@ -256,7 +264,14 @@ func TestMultiProject(t *testing.T) { logger := zerolog.New(io.Discard) - moduleLoader := NewModuleLoader(path, NewSharedHCLParser(), &CredentialsSource{FetchToken: credentials.FindTerraformCloudToken}, config.TerraformSourceMap{}, logger, &sync2.KeyMutex{}) + moduleLoader := NewModuleLoader(ModuleLoaderOptions{ + CachePath: path, + HCLParser: NewSharedHCLParser(), + CredentialsSource: &CredentialsSource{FetchToken: credentials.FindTerraformCloudToken}, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync2.KeyMutex{}, + }) wg := &sync.WaitGroup{} wg.Add(3) diff --git a/internal/hcl/modules/s3_cache.go b/internal/hcl/modules/s3_cache.go new file mode 100644 index 00000000000..11fb7a7bbd0 --- /dev/null +++ b/internal/hcl/modules/s3_cache.go @@ -0,0 +1,152 @@ +package modules + +import ( + "fmt" + "io" + "net/url" + "os" + "path/filepath" + + "github.com/google/uuid" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/mholt/archiver/v3" +) + +type S3Cache struct { + s3Client *s3.S3 + bucketName string + prefix string +} + +// NewS3Cache creates a new S3Cache instance +func NewS3Cache(region, bucketName, prefix string) (*S3Cache, error) { + sess, err := session.NewSessionWithOptions(session.Options{ + SharedConfigState: session.SharedConfigEnable, + Config: aws.Config{ + Region: aws.String(region), + }, + }) + if err != nil { + return nil, fmt.Errorf("failed to create AWS session: %w", err) + } + + return &S3Cache{ + s3Client: s3.New(sess), + bucketName: bucketName, + prefix: prefix, + }, nil +} + +func (cache *S3Cache) applyPrefix(key string) string { + // URL encode the key first since the module address contains /'s + // and these create folders in S3 + encodedKey := url.QueryEscape(key) + + if cache.prefix != "" { + return fmt.Sprintf("%s/%s", cache.prefix, encodedKey) + } + return encodedKey +} + +// Exists checks if the key exists in the S3 bucket +func (cache *S3Cache) Exists(key string) (bool, error) { + prefixedKey := cache.applyPrefix(key) + _, err := cache.s3Client.HeadObject(&s3.HeadObjectInput{ + Bucket: aws.String(cache.bucketName), + Key: aws.String(prefixedKey), + }) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + switch awsErr.Code() { + case s3.ErrCodeNoSuchKey, "NotFound": + return false, nil + } + } + return false, err + } + return true, nil +} + +// Get downloads the key from the S3 bucket to the destPath +func (cache *S3Cache) Get(key, destPath string) error { + prefixedKey := cache.applyPrefix(key) + + // Download from S3 + result, err := cache.s3Client.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(cache.bucketName), + Key: aws.String(prefixedKey), + }) + if err != nil { + return fmt.Errorf("failed to download from S3: %w", err) + } + defer result.Body.Close() + + // Create a temporary file for the downloaded archive + tmpFile, err := os.CreateTemp("", fmt.Sprintf("s3cache-%s.tar.gz", uuid.New().String())) + if err != nil { + return fmt.Errorf("failed to create temporary file: %w", err) + } + defer os.Remove(tmpFile.Name()) + defer tmpFile.Close() + + // Copy the S3 object to the temporary file + if _, err := io.Copy(tmpFile, result.Body); err != nil { + return fmt.Errorf("failed to save downloaded file: %w", err) + } + + // Extract using archiver + tgz := archiver.NewTarGz() + if err := tgz.Unarchive(tmpFile.Name(), destPath); err != nil { + return fmt.Errorf("failed to extract archive: %w", err) + } + + return nil +} + +// Put uploads the srcPath to the S3 bucket with the key +func (cache *S3Cache) Put(key, srcPath string) error { + prefixedKey := cache.applyPrefix(key) + + // Generate a temporary file path without creating the file + tmpPath := filepath.Join(os.TempDir(), fmt.Sprintf("s3cache-%s.tar.gz", uuid.New().String())) + defer os.Remove(tmpPath) + + tgz := archiver.NewTarGz() + + // Get the contents of the source directory and create a list of paths to archive + entries, err := os.ReadDir(srcPath) + if err != nil { + return fmt.Errorf("failed to read source directory: %w", err) + } + + sources := make([]string, 0, len(entries)) + for _, entry := range entries { + sources = append(sources, filepath.Join(srcPath, entry.Name())) + } + + if err := tgz.Archive(sources, tmpPath); err != nil { + return fmt.Errorf("failed to create archive: %w", err) + } + + // Upload the archive to S3 + file, err := os.Open(tmpPath) + if err != nil { + return fmt.Errorf("failed to open archive: %w", err) + } + defer file.Close() + + _, err = cache.s3Client.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(cache.bucketName), + Key: aws.String(prefixedKey), + Body: file, + }) + if err != nil { + return fmt.Errorf("failed to upload to S3: %w", err) + } + + return nil +} diff --git a/internal/hcl/parser_test.go b/internal/hcl/parser_test.go index 699a7c1f910..bc2e942a9ff 100644 --- a/internal/hcl/parser_test.go +++ b/internal/hcl/parser_test.go @@ -55,7 +55,14 @@ data "cats_cat" "the-cats-mother" { `) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), @@ -153,7 +160,14 @@ output "loadbalancer" { `) logger := newDiscardLogger() - parser := NewParser(RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}), logger) + parser := NewParser(RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }), logger) module, err := parser.ParseDirectory() require.NoError(t, err) @@ -225,7 +239,14 @@ output "exp2" { `) logger := newDiscardLogger() - parser := NewParser(RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}), logger) + parser := NewParser(RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }), logger) module, err := parser.ParseDirectory() require.NoError(t, err) @@ -272,7 +293,14 @@ output "instances" { `) logger := newDiscardLogger() - parser := NewParser(RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}), logger) + parser := NewParser(RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }), logger) module, err := parser.ParseDirectory() require.NoError(t, err) @@ -311,7 +339,14 @@ resource "other_resource" "test" { `) logger := newDiscardLogger() - parser := NewParser(RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}), logger) + parser := NewParser(RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }), logger) module, err := parser.ParseDirectory() require.NoError(t, err) @@ -356,7 +391,14 @@ output "attr_not_exists" { `) logger := newDiscardLogger() - parser := NewParser(RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}), logger) + parser := NewParser(RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }), logger) module, err := parser.ParseDirectory() require.NoError(t, err) @@ -396,7 +438,14 @@ resource "other_resource" "test" { `) logger := newDiscardLogger() - parser := NewParser(RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}), logger) + parser := NewParser(RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }), logger) module, err := parser.ParseDirectory() require.NoError(t, err) @@ -448,7 +497,14 @@ output "serviceendpoint_principals" { `) logger := newDiscardLogger() - parser := NewParser(RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}), logger) + parser := NewParser(RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }), logger) module, err := parser.ParseDirectory() require.NoError(t, err) @@ -486,7 +542,14 @@ output "val" { `) logger := newDiscardLogger() - parser := NewParser(RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}), logger) + parser := NewParser(RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }), logger) module, err := parser.ParseDirectory() require.NoError(t, err) @@ -504,7 +567,14 @@ func Test_SetsHasChangesOnMod(t *testing.T) { path := createTestFile("test.tf", `variable "foo" {}`) logger := newDiscardLogger() - parser := NewParser(RootPath{DetectedPath: filepath.Dir(path), HasChanges: true}, CreateEnvFileMatcher([]string{}, nil), modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}), logger) + parser := NewParser(RootPath{DetectedPath: filepath.Dir(path), HasChanges: true}, CreateEnvFileMatcher([]string{}, nil), modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }), logger) module, err := parser.ParseDirectory() require.NoError(t, err) @@ -526,7 +596,14 @@ output "val" { `) logger := newDiscardLogger() - parser := NewParser(RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}), logger) + parser := NewParser(RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }), logger) module, err := parser.ParseDirectory() require.NoError(t, err) @@ -565,7 +642,14 @@ output "mod_result" { logger := newDiscardLogger() dir := filepath.Dir(path) - loader := modules.NewModuleLoader(dir, modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: dir, + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: path}, CreateEnvFileMatcher([]string{}, nil), @@ -628,7 +712,14 @@ output "mod_result" { ) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: path}, CreateEnvFileMatcher([]string{}, nil), @@ -686,7 +777,14 @@ resource "aws_instance" "my_instance" { `) logger := newDiscardLogger() - parser := NewParser(RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}), logger) + parser := NewParser(RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }), logger) module, err := parser.ParseDirectory() require.NoError(t, err) @@ -827,7 +925,14 @@ resource "test_resource_two" "test" { `) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), @@ -887,7 +992,14 @@ resource "test_resource_two" "test" { `) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), @@ -952,7 +1064,14 @@ output "mod_result" { ) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: path}, CreateEnvFileMatcher([]string{}, nil), @@ -1104,7 +1223,14 @@ output "mod_result" { ) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: path}, CreateEnvFileMatcher([]string{}, nil), @@ -1190,7 +1316,14 @@ resource "dynamic" "resource" { ) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: path}, CreateEnvFileMatcher([]string{}, nil), @@ -1255,7 +1388,14 @@ resource "azurerm_linux_function_app" "function" { ) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), @@ -1309,7 +1449,14 @@ resource "test_resource" "second" { `) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), @@ -1365,7 +1512,14 @@ data "google_compute_zones" "us" { `) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), @@ -1420,7 +1574,14 @@ data "aws_availability_zones" "ne" { `) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), @@ -1471,7 +1632,14 @@ resource "random_shuffle" "bad" { `) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), @@ -1528,7 +1696,14 @@ data "aws_subnets" "example" { }`) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), @@ -1575,7 +1750,14 @@ locals { `) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), @@ -1630,7 +1812,14 @@ resource "baz" "bat" { `) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), @@ -1693,7 +1882,14 @@ resource "aws_instance" "example" { ) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: path}, @@ -1748,7 +1944,14 @@ resource "aws_instance" "example" { ) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: path}, CreateEnvFileMatcher([]string{}, nil), @@ -1786,7 +1989,14 @@ resource "aws_instance" "example" { ) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), @@ -1834,7 +2044,14 @@ resource "aws_instance" "example" { ) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), @@ -1871,7 +2088,14 @@ resource "aws_instance" "example" { ) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: filepath.Dir(path)}, @@ -1901,7 +2125,14 @@ resource "bar" "a" { ) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), @@ -1929,7 +2160,14 @@ resource "time_static" "default" {} ) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), @@ -1984,7 +2222,14 @@ resource "aws_instance" "example" { `) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), @@ -2019,7 +2264,14 @@ func Test_TFJSONAttributes(t *testing.T) { `) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), @@ -2045,7 +2297,14 @@ func BenchmarkParserEvaluate(b *testing.B) { logger := newDiscardLogger() dir := "./testdata/benchmarks/heavy" source, _ := modules.NewTerraformCredentialsSource(modules.BaseCredentialSet{}) - loader := modules.NewModuleLoader(dir, modules.NewSharedHCLParser(), source, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: dir, + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: source, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: dir}, CreateEnvFileMatcher([]string{}, nil), @@ -2089,7 +2348,14 @@ terraform { `) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), @@ -2239,7 +2505,14 @@ resource "aws_instance" "blah" { path := createTestFile("test.tf", tt.content) logger := newDiscardLogger() - loader := modules.NewModuleLoader(filepath.Dir(path), modules.NewSharedHCLParser(), nil, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}) + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: filepath.Dir(path), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: nil, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }) parser := NewParser( RootPath{DetectedPath: filepath.Dir(path)}, CreateEnvFileMatcher([]string{}, nil), diff --git a/internal/providers/terraform/hcl_provider.go b/internal/providers/terraform/hcl_provider.go index 5e1f2069d09..82f4e9f2c5a 100644 --- a/internal/providers/terraform/hcl_provider.go +++ b/internal/providers/terraform/hcl_provider.go @@ -160,7 +160,25 @@ func NewHCLProvider(ctx *config.ProjectContext, rootPath hcl.RootPath, config *H } } - loader := modules.NewModuleLoader(ctx.RunContext.Config.CachePath(), modules.NewSharedHCLParser(), credsSource, ctx.RunContext.Config.TerraformSourceMap, logger, ctx.RunContext.ModuleMutex) + var remoteCache modules.RemoteCache + if runCtx.Config.S3ModuleCacheRegion != "" && runCtx.Config.S3ModuleCacheBucket != "" { + s3ModuleCache, err := modules.NewS3Cache(runCtx.Config.S3ModuleCacheRegion, runCtx.Config.S3ModuleCacheBucket, runCtx.Config.S3ModuleCachePrefix) + if err != nil { + logger.Warn().Msgf("failed to initialize S3 module cache: %s", err) + } else { + remoteCache = s3ModuleCache + } + } + + loader := modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: runCtx.Config.CachePath(), + HCLParser: modules.NewSharedHCLParser(), + CredentialsSource: credsSource, + SourceMap: runCtx.Config.TerraformSourceMap, + Logger: logger, + ModuleSync: runCtx.ModuleMutex, + RemoteCache: remoteCache, + }) cachePath := ctx.RunContext.Config.CachePath() initialPath := rootPath.DetectedPath rootPath.DetectedPath = initialPath diff --git a/internal/providers/terraform/hcl_provider_test.go b/internal/providers/terraform/hcl_provider_test.go index d8b7e2dd0f6..3ccd2f52a22 100644 --- a/internal/providers/terraform/hcl_provider_test.go +++ b/internal/providers/terraform/hcl_provider_test.go @@ -226,7 +226,14 @@ func TestHCLProvider_LoadPlanJSON(t *testing.T) { parser := hcl.NewParser( mods[0], hcl.CreateEnvFileMatcher([]string{}, nil), - modules.NewModuleLoader(startingPath, moduleParser, &modules.CredentialsSource{FetchToken: credentials.FindTerraformCloudToken}, config.TerraformSourceMap{}, logger, &sync.KeyMutex{}), + modules.NewModuleLoader(modules.ModuleLoaderOptions{ + CachePath: startingPath, + HCLParser: moduleParser, + CredentialsSource: &modules.CredentialsSource{FetchToken: credentials.FindTerraformCloudToken}, + SourceMap: config.TerraformSourceMap{}, + Logger: logger, + ModuleSync: &sync.KeyMutex{}, + }), logger, options..., ) From 1697013fecdaf264e94c4fc17d4b0a8887524c73 Mon Sep 17 00:00:00 2001 From: Vadim Golub Date: Wed, 13 Nov 2024 17:36:05 +0100 Subject: [PATCH 37/81] fix: Fix config generator when env name is in the project path IC-1911 (#3243) * fix: Add a test case for config generator When an env name is in the path already, we should ignore the other envs. * fix: Detect env name in project path --- .../env_names_in_path/expected.golden | 19 +++++++++++++++ .../env_names_in_path/infracost.yml.tmpl | 16 +++++++++++++ .../generate/env_names_in_path/tree.txt | 15 ++++++++++++ .../expected.golden | 4 ++-- internal/hcl/parser.go | 13 ++++++++--- internal/hcl/project_locator.go | 23 +++++++++++++++++-- internal/providers/detect.go | 2 +- 7 files changed, 84 insertions(+), 8 deletions(-) create mode 100644 cmd/infracost/testdata/generate/env_names_in_path/expected.golden create mode 100644 cmd/infracost/testdata/generate/env_names_in_path/infracost.yml.tmpl create mode 100644 cmd/infracost/testdata/generate/env_names_in_path/tree.txt diff --git a/cmd/infracost/testdata/generate/env_names_in_path/expected.golden b/cmd/infracost/testdata/generate/env_names_in_path/expected.golden new file mode 100644 index 00000000000..da178b5aa8a --- /dev/null +++ b/cmd/infracost/testdata/generate/env_names_in_path/expected.golden @@ -0,0 +1,19 @@ +version: 0.1 +autodetect: + env_names: + - dev + - prod + - qa + +projects: + - path: infra/components/foo-dev + name: infra-components-foo-dev + terraform_var_files: + - ../../variables/defaults.tfvars + - ../../variables/dev/foo-dev.tfvars + - path: infra/components/foo-prod + name: infra-components-foo-prod + terraform_var_files: + - ../../variables/defaults.tfvars + - ../../variables/prod/foo-prod.tfvars + diff --git a/cmd/infracost/testdata/generate/env_names_in_path/infracost.yml.tmpl b/cmd/infracost/testdata/generate/env_names_in_path/infracost.yml.tmpl new file mode 100644 index 00000000000..0cdc6733793 --- /dev/null +++ b/cmd/infracost/testdata/generate/env_names_in_path/infracost.yml.tmpl @@ -0,0 +1,16 @@ +version: 0.1 +autodetect: + env_names: + - dev + - prod + - qa + +projects: +{{- range $project := .DetectedProjects }} + - path: {{ $project.Path }} + name: {{ $project.Name }} + terraform_var_files: + {{- range $varFile := $project.TerraformVarFiles }} + - {{ $varFile }} + {{- end }} +{{- end }} diff --git a/cmd/infracost/testdata/generate/env_names_in_path/tree.txt b/cmd/infracost/testdata/generate/env_names_in_path/tree.txt new file mode 100644 index 00000000000..c759b71ed90 --- /dev/null +++ b/cmd/infracost/testdata/generate/env_names_in_path/tree.txt @@ -0,0 +1,15 @@ +. +└── infra + ├── components + │ ├── foo-dev + │ │ └── main.tf + │ └── foo-prod + │ └── main.tf + └── variables + ├── dev + │ └── foo-dev.tfvars + ├── prod + │ └── foo-prod.tfvars + ├── qa + │ └── bar.tfvars + └── defaults.tfvars diff --git a/cmd/infracost/testdata/generate/modules_and_external_tfvars/expected.golden b/cmd/infracost/testdata/generate/modules_and_external_tfvars/expected.golden index 9d2fad87855..9ee9ffa998e 100755 --- a/cmd/infracost/testdata/generate/modules_and_external_tfvars/expected.golden +++ b/cmd/infracost/testdata/generate/modules_and_external_tfvars/expected.golden @@ -2,7 +2,7 @@ version: 0.1 projects: - path: db/dev - name: db-dev-dev + name: db-dev terraform_var_files: - ../../default.tfvars - ../../envs/dev.tfvars @@ -13,7 +13,7 @@ projects: - default.tfvars - envs/dev.tfvars - path: db/prod - name: db-prod-prod + name: db-prod terraform_var_files: - ../../default.tfvars - ../../envs/prod.tfvars diff --git a/internal/hcl/parser.go b/internal/hcl/parser.go index 0d740b22d17..381476f3dbd 100644 --- a/internal/hcl/parser.go +++ b/internal/hcl/parser.go @@ -108,10 +108,17 @@ func makePathsRelativeToInitial(paths []string, initialPath string) []string { } // OptionWithModuleSuffix sets an optional module suffix which will be added to the Module after it has finished parsing -// this can be used to augment auto-detected project path names and metadata. -func OptionWithModuleSuffix(suffix string) Option { +// this can be used to augment auto-detected project path names and metadata. If the suffix is already part of the project name - ignore it. +func OptionWithModuleSuffix(rootPath, suffix string) Option { return func(p *Parser) { - p.moduleSuffix = suffix + pathEnv := "" + if p.envMatcher != nil { + pathEnv = p.envMatcher.PathEnv(rootPath) + } + + if pathEnv == "" || (suffix != "" && pathEnv != suffix) { + p.moduleSuffix = suffix + } } } diff --git a/internal/hcl/project_locator.go b/internal/hcl/project_locator.go index 03b3a666efb..f482526dfae 100644 --- a/internal/hcl/project_locator.go +++ b/internal/hcl/project_locator.go @@ -251,6 +251,18 @@ func (e *EnvFileMatcher) EnvName(file string) string { return clean } +// PathEnv returns the env name detected in file path when it matches defined envs. Otherwise returns an empty string. +func (e *EnvFileMatcher) PathEnv(file string) string { + env := e.EnvName(file) + + _, ok := e.envLookup[env] + if ok { + return env + } + + return "" +} + func (e *EnvFileMatcher) hasEnvPrefix(clean string, name string) bool { return strings.HasPrefix(clean, name+"-") || strings.HasPrefix(clean, name+"_") || strings.HasPrefix(clean, name+".") } @@ -961,8 +973,8 @@ func (t *TreeNode) CollectRootPaths(e *EnvFileMatcher) []RootPath { found := make(map[string]bool) for _, root := range projects { + base := filepath.Base(root.DetectedPath) for _, varFile := range root.TerraformVarFiles { - base := filepath.Base(root.DetectedPath) name := e.clean(varFile.Name) if base == name { found[varFile.FullPath] = true @@ -975,10 +987,11 @@ func (t *TreeNode) CollectRootPaths(e *EnvFileMatcher) []RootPath { // terraform var files that are scoped to a specific project // are not added to another project. for i, root := range projects { + base := filepath.Base(root.DetectedPath) + var filtered RootPathVarFiles for _, varFile := range root.TerraformVarFiles { name := e.clean(varFile.Name) - base := filepath.Base(root.DetectedPath) if found[varFile.FullPath] && base != name { continue } @@ -1081,6 +1094,7 @@ func (r *RootPath) EnvGroupings() []VarFileGrouping { } } + pathEnv := r.Matcher.PathEnv(r.DetectedPath) hasChildVarFileEnvs := len(varFileGrouping) > 0 for _, varFile := range varFiles { @@ -1090,6 +1104,11 @@ func (r *RootPath) EnvGroupings() []VarFileGrouping { env := r.Matcher.EnvName(varFile.EnvName) _, exists := varFileGrouping[env] + + // When env is a part of the project name, we should ignore any other detected envs + if pathEnv != "" && env != pathEnv { + continue + } // only add the non child env var files if there are no envs defined that are // closer to the project, or if the env matches one defined as a child var file. if !hasChildVarFileEnvs || (hasChildVarFileEnvs && exists) { diff --git a/internal/providers/detect.go b/internal/providers/detect.go index c3971f32901..069db703a9c 100644 --- a/internal/providers/detect.go +++ b/internal/providers/detect.go @@ -159,7 +159,7 @@ func autodetectedRootToProviders(projectContext *config.ProjectContext, rootPath append( options, hcl.OptionWithTFVarsPaths(append(autoVarFiles.ToPaths(), env.TerraformVarFiles.ToPaths()...), true), - hcl.OptionWithModuleSuffix(env.Name), + hcl.OptionWithModuleSuffix(rootPath.DetectedPath, env.Name), )...) if err != nil { logging.Logger.Warn().Err(err).Msgf("could not initialize provider for path %q", rootPath.DetectedPath) From e8de081c07b5488433da2e28cef90f78341ffafa Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Wed, 13 Nov 2024 18:08:56 +0000 Subject: [PATCH 38/81] enhance: add different TTLs for module branches --- internal/hcl/modules/fetch.go | 35 ++++++++++++++++++++++++++++-- internal/hcl/modules/fetch_test.go | 34 +++++++++++++++++++++-------- internal/hcl/modules/loader.go | 3 ++- internal/hcl/modules/s3_cache.go | 31 +++++++++++++++++++++----- 4 files changed, 86 insertions(+), 17 deletions(-) diff --git a/internal/hcl/modules/fetch.go b/internal/hcl/modules/fetch.go index ab43cf49a3d..072a6accb95 100644 --- a/internal/hcl/modules/fetch.go +++ b/internal/hcl/modules/fetch.go @@ -5,8 +5,10 @@ import ( "net/url" "os" "path/filepath" + "regexp" "strings" "sync" + "time" getter "github.com/hashicorp/go-getter" "github.com/otiai10/copy" @@ -15,6 +17,11 @@ import ( "github.com/infracost/infracost/internal/logging" ) +var tagRegex = regexp.MustCompile(`^v?\d+\.\d+\.\d+`) +var commitRegex = regexp.MustCompile(`^([0-9a-f]{40})|([0-9a-f]{7})$`) +var defaultTTL = 24 * time.Hour +var tagCommitTTL = 30 * 24 * time.Hour + // PackageFetcher downloads modules from a remote source to the given destination // This supports all the non-local and non-Terraform registry sources listed here: https://www.terraform.io/language/modules/sources type PackageFetcher struct { @@ -65,8 +72,9 @@ func (p *PackageFetcher) fetch(moduleAddr string, dest string) error { p.localCache.Store(moduleAddr, dest) if p.remoteCache != nil { - p.logger.Debug().Msgf("putting module %s into remote cache", moduleAddr) - err = p.remoteCache.Put(moduleAddr, dest) + ttl := determineTTL(moduleAddr) + p.logger.Debug().Msgf("putting module %s into remote cache with ttl %s", moduleAddr, ttl) + err = p.remoteCache.Put(moduleAddr, dest, ttl) if err != nil { p.logger.Warn().Msgf("error putting module %s into remote cache: %s", moduleAddr, err) } @@ -160,6 +168,29 @@ func (p *PackageFetcher) fetchFromRemote(moduleAddr, dest string) (bool, error) return true, nil } +func determineTTL(moduleAddr string) time.Duration { + u, err := url.Parse(moduleAddr) + if err != nil { + return defaultTTL + } + + // Get the ref parameter + ref := u.Query().Get("ref") + if ref == "" { + return defaultTTL + } + + // Check if ref looks like a git tag or a commit + isTag := tagRegex.MatchString(ref) + isCommit := commitRegex.MatchString(ref) + + if isTag || isCommit { + return tagCommitTTL + } + + return defaultTTL +} + // CustomGitGetter extends the standard GitGetter transforming SSH sources to // HTTPs first before attempting a Get. This means that we can attempt to use any // Git credentials on the host machine to resolve the Get before falling back to diff --git a/internal/hcl/modules/fetch_test.go b/internal/hcl/modules/fetch_test.go index 79345bba939..22911c8d00b 100644 --- a/internal/hcl/modules/fetch_test.go +++ b/internal/hcl/modules/fetch_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "sync" "testing" + "time" getter "github.com/hashicorp/go-getter" "github.com/rs/zerolog" @@ -40,7 +41,7 @@ func TestPackageFetcher_fetch_RemoteCache(t *testing.T) { logger := zerolog.New(io.Discard) mock := &mockRemoteCache{ - cache: make(map[string]string), + cache: make(map[string]mockCacheEntry), } tests := []struct { @@ -62,7 +63,7 @@ func TestPackageFetcher_fetch_RemoteCache(t *testing.T) { } `), 0600)) // Store the module directory path in the cache - require.NoError(t, c.Put("git::https://github.com/terraform-aws-modules/terraform-aws-vpc?ref=v5.15.0", moduleDir)) + require.NoError(t, c.Put("git::https://github.com/terraform-aws-modules/terraform-aws-vpc?ref=v5.15.0", moduleDir, 24*time.Hour)) }, expectedCalls: map[string]int{ "Exists": 1, @@ -124,8 +125,13 @@ func TestPackageFetcher_fetch_RemoteCache(t *testing.T) { } } +type mockCacheEntry struct { + path string + expiresAt time.Time +} + type mockRemoteCache struct { - cache map[string]string + cache map[string]mockCacheEntry existsCalls int getCalls int putCalls int @@ -142,7 +148,14 @@ func (m *mockRemoteCache) Exists(key string) (bool, error) { return false, fmt.Errorf("mock remote cache error") } - _, exists := m.cache[key] + entry, exists := m.cache[key] + + // Check if the entry has expired + if time.Now().After(entry.expiresAt) { + delete(m.cache, key) + return false, nil + } + return exists, nil } @@ -155,14 +168,14 @@ func (m *mockRemoteCache) Get(key string, dest string) error { return fmt.Errorf("mock remote cache error") } - srcDir, exists := m.cache[key] + entry, exists := m.cache[key] if !exists { return fmt.Errorf("key not found: %s", key) } // Use go-getter to copy the directory client := &getter.Client{ - Src: fmt.Sprintf("file://%s", srcDir), + Src: fmt.Sprintf("file://%s", entry.path), Dst: dest, Mode: getter.ClientModeDir, } @@ -170,7 +183,7 @@ func (m *mockRemoteCache) Get(key string, dest string) error { return client.Get() } -func (m *mockRemoteCache) Put(key string, src string) error { +func (m *mockRemoteCache) Put(key string, src string, ttl time.Duration) error { m.mu.Lock() defer m.mu.Unlock() m.putCalls++ @@ -179,14 +192,17 @@ func (m *mockRemoteCache) Put(key string, src string) error { return fmt.Errorf("mock remote cache error") } - m.cache[key] = src + m.cache[key] = mockCacheEntry{ + path: src, + expiresAt: time.Now().Add(ttl), + } return nil } func (m *mockRemoteCache) ResetCache() { m.mu.Lock() defer m.mu.Unlock() - m.cache = make(map[string]string) + m.cache = make(map[string]mockCacheEntry) } func (m *mockRemoteCache) ResetCounters() { diff --git a/internal/hcl/modules/loader.go b/internal/hcl/modules/loader.go index 7251db5f1c9..a57ddfe08d6 100644 --- a/internal/hcl/modules/loader.go +++ b/internal/hcl/modules/loader.go @@ -14,6 +14,7 @@ import ( "sort" "strings" "sync" + "time" getter "github.com/hashicorp/go-getter" "github.com/hashicorp/hcl/v2" @@ -47,7 +48,7 @@ var ( type RemoteCache interface { Exists(key string) (bool, error) Get(key string, dest string) error - Put(key string, src string) error + Put(key string, src string, ttl time.Duration) error } // ModuleLoader handles the loading of Terraform modules. It supports local, registry and other remote modules. diff --git a/internal/hcl/modules/s3_cache.go b/internal/hcl/modules/s3_cache.go index 11fb7a7bbd0..a9a227da81d 100644 --- a/internal/hcl/modules/s3_cache.go +++ b/internal/hcl/modules/s3_cache.go @@ -6,6 +6,7 @@ import ( "net/url" "os" "path/filepath" + "time" "github.com/google/uuid" @@ -55,7 +56,7 @@ func (cache *S3Cache) applyPrefix(key string) string { // Exists checks if the key exists in the S3 bucket func (cache *S3Cache) Exists(key string) (bool, error) { prefixedKey := cache.applyPrefix(key) - _, err := cache.s3Client.HeadObject(&s3.HeadObjectInput{ + headObj, err := cache.s3Client.HeadObject(&s3.HeadObjectInput{ Bucket: aws.String(cache.bucketName), Key: aws.String(prefixedKey), }) @@ -68,6 +69,19 @@ func (cache *S3Cache) Exists(key string) (bool, error) { } return false, err } + + // Check expiration based on x-amz-meta-expires-at + if expiresAtStr, ok := headObj.Metadata["x-amz-meta-expires-at"]; ok { + expirationTime, err := time.Parse(time.RFC3339, *expiresAtStr) + if err != nil { + return false, fmt.Errorf("failed to parse expiration metadata: %w", err) + } + + if time.Now().After(expirationTime) { + return false, nil // Object is expired + } + } + return true, nil } @@ -108,7 +122,7 @@ func (cache *S3Cache) Get(key, destPath string) error { } // Put uploads the srcPath to the S3 bucket with the key -func (cache *S3Cache) Put(key, srcPath string) error { +func (cache *S3Cache) Put(key, srcPath string, ttl time.Duration) error { prefixedKey := cache.applyPrefix(key) // Generate a temporary file path without creating the file @@ -139,10 +153,17 @@ func (cache *S3Cache) Put(key, srcPath string) error { } defer file.Close() + // Calculate expiration time and set it as metadata + expirationTime := time.Now().Add(ttl).Format(time.RFC3339) + metadata := map[string]*string{ + "x-amz-meta-expires-at": aws.String(expirationTime), + } + _, err = cache.s3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(cache.bucketName), - Key: aws.String(prefixedKey), - Body: file, + Bucket: aws.String(cache.bucketName), + Key: aws.String(prefixedKey), + Body: file, + Metadata: metadata, }) if err != nil { return fmt.Errorf("failed to upload to S3: %w", err) From a8f2ad7afb70166b1704a429fd7c3150cd993615 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Wed, 13 Nov 2024 19:53:44 +0000 Subject: [PATCH 39/81] enhance: update remote cache to work with sparse checkout --- go.sum | 8 +- internal/hcl/modules/fetch.go | 40 +++++--- internal/hcl/modules/loader.go | 98 ++++++++++++++++--- internal/hcl/modules/s3_cache.go | 1 + .../terraform/terragrunt_hcl_provider.go | 35 +++++-- 5 files changed, 141 insertions(+), 41 deletions(-) diff --git a/go.sum b/go.sum index 09c6b51d5ef..84084349903 100644 --- a/go.sum +++ b/go.sum @@ -818,10 +818,10 @@ github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/infracost/go-getter v0.0.0-20240909111353-c0d2eebadfd5 h1:FiK2b8h6CezRGI6CGs7YDVG9nbF2TQGMcRK9iSM37so= -github.com/infracost/go-getter v0.0.0-20240909111353-c0d2eebadfd5/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= -github.com/infracost/terragrunt v0.47.1-0.20240920091357-980f132ffa5a h1:S24y2y7hdsgtT4HD/dUAiangWfPv4cz+n3f4/J2di0M= -github.com/infracost/terragrunt v0.47.1-0.20240920091357-980f132ffa5a/go.mod h1:504J5iD5AjGgP5IwHkUWAcfoqvZIhrR1aEvyxDxX1VA= +github.com/infracost/go-getter v0.0.0-20241113193122-eeeb4e90c01c h1:ZDJg7/oQDwKWApxwWDZb98DMOoaPkaFdmZD93EZupu8= +github.com/infracost/go-getter v0.0.0-20241113193122-eeeb4e90c01c/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= +github.com/infracost/terragrunt v0.47.1-0.20241113223044-d35392684463 h1:vE7CFFOLqsCgYjeUJElbt5MoWEkWIZxxXGK2Wu1klu4= +github.com/infracost/terragrunt v0.47.1-0.20241113223044-d35392684463/go.mod h1:504J5iD5AjGgP5IwHkUWAcfoqvZIhrR1aEvyxDxX1VA= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= diff --git a/internal/hcl/modules/fetch.go b/internal/hcl/modules/fetch.go index 072a6accb95..b51ff28affc 100644 --- a/internal/hcl/modules/fetch.go +++ b/internal/hcl/modules/fetch.go @@ -10,6 +10,7 @@ import ( "sync" "time" + tgterraform "github.com/gruntwork-io/terragrunt/terraform" getter "github.com/hashicorp/go-getter" "github.com/otiai10/copy" "github.com/rs/zerolog" @@ -40,30 +41,32 @@ func NewPackageFetcher(remoteCache RemoteCache, logger zerolog.Logger) *PackageF // fetch downloads the remote module using the go-getter library // See: https://github.com/hashicorp/go-getter -func (p *PackageFetcher) fetch(moduleAddr string, dest string) error { +func (p *PackageFetcher) Fetch(moduleAddr string, dest string) error { fetched, err := p.fetchFromLocalCache(moduleAddr, dest) if fetched { - p.logger.Debug().Msgf("fetched module %s from local cache", moduleAddr) + p.logger.Trace().Msgf("cache hit (local): %s", moduleAddr) + p.logger.Info().Msgf("cache hit (local): %s", moduleAddr) return nil } if err != nil { - p.logger.Warn().Msgf("error fetching module %s from local cache, trying to fetch from elsewhere: %s", moduleAddr, err) + p.logger.Warn().Msgf("error fetching module %s from local cache: %s", moduleAddr, err) } - if p.remoteCache != nil { - fetched, err = p.fetchFromRemoteCache(moduleAddr, dest) - if fetched { - p.logger.Debug().Msgf("fetched module %s from remote cache", moduleAddr) - p.localCache.Store(moduleAddr, dest) - return nil - } + fetched, err = p.fetchFromRemoteCache(moduleAddr, dest) + if fetched { + p.logger.Trace().Msgf("cache hit (remote): %s", moduleAddr) + p.localCache.Store(moduleAddr, dest) + return nil + } - if err != nil { - p.logger.Warn().Msgf("error fetching module %s from remote cache, trying to fetch from elsewhere: %s", moduleAddr, err) - } + if err != nil { + p.logger.Warn().Msgf("error fetching module %s from remote cache: %s", moduleAddr, err) } + p.logger.Trace().Msgf("cache miss: %s", moduleAddr) + p.logger.Info().Msgf("cache miss: %s", moduleAddr) + _, err = p.fetchFromRemote(moduleAddr, dest) if err != nil { return fmt.Errorf("error fetching module %s from remote: %w", moduleAddr, err) @@ -91,6 +94,10 @@ func (p *PackageFetcher) fetchFromLocalCache(moduleAddr, dest string) (bool, err prevDest, _ := v.(string) + if prevDest == dest { + return true, nil + } + p.logger.Debug().Msgf("module %s already downloaded, copying from '%s' to '%s'", moduleAddr, prevDest, dest) err := os.Mkdir(dest, os.ModePerm) @@ -117,6 +124,10 @@ func (p *PackageFetcher) fetchFromLocalCache(moduleAddr, dest string) (bool, err } func (p *PackageFetcher) fetchFromRemoteCache(moduleAddr, dest string) (bool, error) { + if p.remoteCache == nil { + return false, nil + } + ok, err := p.remoteCache.Exists(moduleAddr) if err != nil { return false, err @@ -149,6 +160,9 @@ func (p *PackageFetcher) fetchFromRemote(moduleAddr, dest string) (bool, error) getters[k] = g } + // This is a custom getter used by Terragrunt + getters["tfr"] = &tgterraform.RegistryGetter{} + getters["git"] = &CustomGitGetter{new(getter.GitGetter)} client := getter.Client{ diff --git a/internal/hcl/modules/loader.go b/internal/hcl/modules/loader.go index a57ddfe08d6..5df685fb2fc 100644 --- a/internal/hcl/modules/loader.go +++ b/internal/hcl/modules/loader.go @@ -19,6 +19,7 @@ import ( getter "github.com/hashicorp/go-getter" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/terraform-config-inspect/tfconfig" + "github.com/otiai10/copy" "github.com/rs/zerolog" "golang.org/x/sync/errgroup" @@ -364,7 +365,7 @@ func (m *ModuleLoader) loadModule(moduleCall *tfconfig.ModuleCall, parentPath st } // checkoutPathIfRequired checks if the given directories are in the sparse-checkout file list and adds them if not. -func (m *ModuleLoader) checkoutPathIfRequired(repoRoot string, dirs string) error { +func (m *ModuleLoader) checkoutPathIfRequired(repoRoot string, dir string) error { // Lock the git repo root so we don't have multiple calls trying to read and update the sparse-checkout file list. unlock := m.sync.Lock(repoRoot) defer unlock() @@ -384,22 +385,22 @@ func (m *ModuleLoader) checkoutPathIfRequired(repoRoot string, dirs string) erro m.logger.Trace().Msgf("getting sparse checkout directories for path %s", repoRoot) existingDirs, err := getSparseCheckoutDirs(repoRoot) if err != nil { - // If the error indicates that sparse checkout is not enabled, just return nil - if err.Error() == "sparse-checkout not enabled" { - m.logger.Trace().Msgf("sparse-checkout not enabled for path %s", repoRoot) - return nil - } + return err + } + + sourceURL, err := getGitURL(repoRoot) + if err != nil { return err } mu := &sync.Mutex{} - return RecursivelyAddDirsToSparseCheckout(repoRoot, existingDirs, []string{dirs}, mu, m.logger, 0) + return RecursivelyAddDirsToSparseCheckout(repoRoot, sourceURL, m.packageFetcher, existingDirs, []string{dir}, mu, m.logger, 0) } // RecursivelyAddDirsToSparseCheckout adds the given directories to the sparse-checkout file list. // It then checks any symlinks within the directories and adds them to the sparse-checkout file list as well. -func RecursivelyAddDirsToSparseCheckout(repoRoot string, existingDirs []string, dirs []string, mu *sync.Mutex, logger zerolog.Logger, depth int) error { +func RecursivelyAddDirsToSparseCheckout(repoRoot string, sourceURL string, packageFetcher *PackageFetcher, existingDirs []string, dirs []string, mu *sync.Mutex, logger zerolog.Logger, depth int) error { var newDirs []string // Sort the existing directories and dirs to be added by length @@ -424,13 +425,57 @@ func RecursivelyAddDirsToSparseCheckout(repoRoot string, existingDirs []string, return nil } - logger.Trace().Msgf("adding dirs to sparse-checkout for repo %s: %v", repoRoot, newDirs) - mu.Lock() - err := setSparseCheckoutDirs(repoRoot, existingDirs) - mu.Unlock() + parsedSourceURL, err := url.Parse(sourceURL) + if err != nil { + return err + } + + // Create a temporary directory for this fetch + tmpDir, err := os.MkdirTemp(os.TempDir(), "infracost-sparse-checkout") if err != nil { return err } + defer os.RemoveAll(tmpDir) + + logger.Trace().Msgf("adding dirs to sparse-checkout for repo %s: %v", repoRoot, newDirs) + for _, dir := range newDirs { + q := parsedSourceURL.Query() + q.Set("subdir", dir) + parsedSourceURL.RawQuery = q.Encode() + s := parsedSourceURL.String() + + // Load the package either from the cache or by pulling from the remote. + // We load it into a temporary directory so we can then merge it into the repo root. + // If we load it into the repo root then the fetcher will try and cache + // the entire repo root which we don't want. + dirTmpDir := filepath.Join(tmpDir, "fetch-"+filepath.Base(dir)) + err := packageFetcher.Fetch(s, dirTmpDir) + if err != nil { + return err + } + + mu.Lock() + // Copy the downloaded package into the repo root + opt := copy.Options{ + OnSymlink: func(src string) copy.SymlinkAction { + return copy.Shallow + }, + } + err = copy.Copy(dirTmpDir, repoRoot, opt) + if err != nil { + return err + } + + // After we've fetched the package we need to update the sparse checkout list + // because the package fetcher retrieved it from the remote cache this won't + // have been done and future calls to download other subdirs won't work + err = setSparseCheckoutDirs(repoRoot, existingDirs) + mu.Unlock() + + if err != nil { + return err + } + } if depth >= maxSparseCheckoutDepth { return nil @@ -452,7 +497,7 @@ func RecursivelyAddDirsToSparseCheckout(repoRoot string, existingDirs []string, if len(additionalDirs) > 0 { logger.Trace().Msgf("recursively adding symlinked dirs to sparse-checkout for repo %s: %v", repoRoot, additionalDirs) - return RecursivelyAddDirsToSparseCheckout(repoRoot, existingDirs, additionalDirs, mu, logger, depth+1) + return RecursivelyAddDirsToSparseCheckout(repoRoot, sourceURL, packageFetcher, existingDirs, additionalDirs, mu, logger, depth+1) } return nil @@ -480,6 +525,31 @@ func findGitRepoRoot(startPath string) (string, error) { return strings.TrimSpace(string(output)), nil } +// getGitURL gets the Git URL for the given path +func getGitURL(path string) (string, error) { + // Get remote + cmd := exec.Command("git", "config", "--get", "remote.origin.url") + cmd.Dir = path + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("error getting git URL: %w", err) + } + + remote := strings.TrimSpace(string(output)) + + // Get commit + cmd = exec.Command("git", "rev-parse", "HEAD") + cmd.Dir = path + output, err = cmd.Output() + if err != nil { + return "", fmt.Errorf("error getting git commit: %w", err) + } + + commit := strings.TrimSpace(string(output)) + + return fmt.Sprintf("git::%s?ref=%s", remote, commit), nil +} + // isSparseCheckoutEnabled checks if sparse-checkout is enabled in the repository func isSparseCheckoutEnabled(repoRoot string) (bool, error) { cmd := exec.Command("git", "config", "--get", "core.sparseCheckout") @@ -723,7 +793,7 @@ func (m *ModuleLoader) loadRemoteModule(key string, source string) (*ManifestMod return manifestModule, nil } - err = m.packageFetcher.fetch(moduleAddr, dest) + err = m.packageFetcher.Fetch(moduleAddr, dest) if err != nil { return nil, schema.NewFailedDownloadDiagnostic(source, err) } diff --git a/internal/hcl/modules/s3_cache.go b/internal/hcl/modules/s3_cache.go index a9a227da81d..d5664cc2cfe 100644 --- a/internal/hcl/modules/s3_cache.go +++ b/internal/hcl/modules/s3_cache.go @@ -114,6 +114,7 @@ func (cache *S3Cache) Get(key, destPath string) error { // Extract using archiver tgz := archiver.NewTarGz() + tgz.OverwriteExisting = true if err := tgz.Unarchive(tmpFile.Name(), destPath); err != nil { return fmt.Errorf("failed to extract archive: %w", err) } diff --git a/internal/providers/terraform/terragrunt_hcl_provider.go b/internal/providers/terraform/terragrunt_hcl_provider.go index 3a8245dff1b..c69c2ad25a7 100644 --- a/internal/providers/terraform/terragrunt_hcl_provider.go +++ b/internal/providers/terraform/terragrunt_hcl_provider.go @@ -107,6 +107,7 @@ type TerragruntHCLProvider struct { excludedPaths []string env map[string]string sourceCache map[string]string + remoteCache modules.RemoteCache logger zerolog.Logger } @@ -117,12 +118,24 @@ func NewTerragruntHCLProvider(rootPath hcl.RootPath, ctx *config.ProjectContext) "provider", "terragrunt_dir", ).Logger() + var remoteCache modules.RemoteCache + runCtx := ctx.RunContext + if runCtx.Config.S3ModuleCacheRegion != "" && runCtx.Config.S3ModuleCacheBucket != "" { + s3ModuleCache, err := modules.NewS3Cache(runCtx.Config.S3ModuleCacheRegion, runCtx.Config.S3ModuleCacheBucket, runCtx.Config.S3ModuleCachePrefix) + if err != nil { + logger.Warn().Msgf("failed to initialize S3 module cache: %s", err) + } else { + remoteCache = s3ModuleCache + } + } + return &TerragruntHCLProvider{ ctx: ctx, Path: rootPath, excludedPaths: ctx.ProjectConfig.ExcludePaths, env: getEnvVars(ctx), sourceCache: map[string]string{}, + remoteCache: remoteCache, logger: logger, } } @@ -564,7 +577,7 @@ func (p *TerragruntHCLProvider) runTerragrunt(opts *tgoptions.TerragruntOptions) return } if sourceURL != "" { - updatedWorkingDir, err := downloadSourceOnce(sourceURL, opts, terragruntConfig, p.logger) + updatedWorkingDir, err := downloadSourceOnce(sourceURL, opts, terragruntConfig, p.remoteCache, p.logger) if err != nil { info.error = err @@ -644,23 +657,24 @@ func splitModuleSubDir(moduleSource string) (string, string, error) { } // downloadSourceOnce thread-safely makes sure the sourceURL is only downloaded once -func downloadSourceOnce(sourceURL string, opts *tgoptions.TerragruntOptions, terragruntConfig *tgconfig.TerragruntConfig, logger zerolog.Logger) (string, error) { +func downloadSourceOnce(sourceURL string, opts *tgoptions.TerragruntOptions, terragruntConfig *tgconfig.TerragruntConfig, remoteCache modules.RemoteCache, logger zerolog.Logger) (string, error) { _, modAddr, err := splitModuleSubDir(sourceURL) if err != nil { return "", err } + parsedSourceURL, err := url.Parse(sourceURL) + if err != nil { + return "", err + } + // If sparse checkout is enabled add the subdir to the Source URL as a query param // so go-getter only downloads the required directory. if os.Getenv("INFRACOST_SPARSE_CHECKOUT") == "true" { - u, err := url.Parse(sourceURL) - if err != nil { - return "", err - } - q := u.Query() + q := parsedSourceURL.Query() q.Set("subdir", modAddr) - u.RawQuery = q.Encode() - sourceURL = u.String() + parsedSourceURL.RawQuery = q.Encode() + sourceURL = parsedSourceURL.String() } source, err := tfsource.NewSource(sourceURL, opts.DownloadDir, opts.WorkingDir, terragruntConfig.GenerateConfigs, opts.Logger) @@ -703,7 +717,8 @@ func downloadSourceOnce(sourceURL string, opts *tgoptions.TerragruntOptions, ter logger.Trace().Msgf("recursively adding symlinked dirs to sparse-checkout for repo %s: %v", dir, symlinkedDirs) // Using a depth of 1 here since the submodule directory is already downloaded, so only need // to add the symlinked directories to the sparse-checkout. - err := modules.RecursivelyAddDirsToSparseCheckout(dir, []string{modAddr}, symlinkedDirs, mu, logger, 1) + packageFetcher := modules.NewPackageFetcher(remoteCache, logger) + err := modules.RecursivelyAddDirsToSparseCheckout(dir, parsedSourceURL, packageFetcher, []string{modAddr}, symlinkedDirs, mu, logger, 1) if err != nil { return "", err } From a1320ff418dac27aea8b50c8afbd46a27c157443 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Wed, 13 Nov 2024 20:28:23 +0000 Subject: [PATCH 40/81] fix: S3 cache should treat forbidden as not found When there's no permissions to ListBucket then retrieving an unknown object will return a forbidden error. --- go.sum | 4 ++-- internal/hcl/modules/fetch.go | 2 -- internal/hcl/modules/loader.go | 8 +++++++- internal/hcl/modules/s3_cache.go | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/go.sum b/go.sum index 84084349903..ca01b47eaea 100644 --- a/go.sum +++ b/go.sum @@ -818,8 +818,8 @@ github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/infracost/go-getter v0.0.0-20241113193122-eeeb4e90c01c h1:ZDJg7/oQDwKWApxwWDZb98DMOoaPkaFdmZD93EZupu8= -github.com/infracost/go-getter v0.0.0-20241113193122-eeeb4e90c01c/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= +github.com/infracost/go-getter v0.0.0-20240909111353-c0d2eebadfd5 h1:FiK2b8h6CezRGI6CGs7YDVG9nbF2TQGMcRK9iSM37so= +github.com/infracost/go-getter v0.0.0-20240909111353-c0d2eebadfd5/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= github.com/infracost/terragrunt v0.47.1-0.20241113223044-d35392684463 h1:vE7CFFOLqsCgYjeUJElbt5MoWEkWIZxxXGK2Wu1klu4= github.com/infracost/terragrunt v0.47.1-0.20241113223044-d35392684463/go.mod h1:504J5iD5AjGgP5IwHkUWAcfoqvZIhrR1aEvyxDxX1VA= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= diff --git a/internal/hcl/modules/fetch.go b/internal/hcl/modules/fetch.go index b51ff28affc..a36d94d5d62 100644 --- a/internal/hcl/modules/fetch.go +++ b/internal/hcl/modules/fetch.go @@ -45,7 +45,6 @@ func (p *PackageFetcher) Fetch(moduleAddr string, dest string) error { fetched, err := p.fetchFromLocalCache(moduleAddr, dest) if fetched { p.logger.Trace().Msgf("cache hit (local): %s", moduleAddr) - p.logger.Info().Msgf("cache hit (local): %s", moduleAddr) return nil } @@ -65,7 +64,6 @@ func (p *PackageFetcher) Fetch(moduleAddr string, dest string) error { } p.logger.Trace().Msgf("cache miss: %s", moduleAddr) - p.logger.Info().Msgf("cache miss: %s", moduleAddr) _, err = p.fetchFromRemote(moduleAddr, dest) if err != nil { diff --git a/internal/hcl/modules/loader.go b/internal/hcl/modules/loader.go index 5df685fb2fc..2e7253aea9b 100644 --- a/internal/hcl/modules/loader.go +++ b/internal/hcl/modules/loader.go @@ -385,7 +385,13 @@ func (m *ModuleLoader) checkoutPathIfRequired(repoRoot string, dir string) error m.logger.Trace().Msgf("getting sparse checkout directories for path %s", repoRoot) existingDirs, err := getSparseCheckoutDirs(repoRoot) if err != nil { - return err + // If the error indicates that sparse checkout is not enabled, just return nil + // Even though we check this above, we need to check it again here because the sparse-checkout + // config might be enabled but sparse-checkout might not be fully initialized + if err.Error() == "sparse-checkout not enabled" { + m.logger.Trace().Msgf("sparse-checkout not enabled for path %s", repoRoot) + return nil + } } sourceURL, err := getGitURL(repoRoot) diff --git a/internal/hcl/modules/s3_cache.go b/internal/hcl/modules/s3_cache.go index d5664cc2cfe..744f851af91 100644 --- a/internal/hcl/modules/s3_cache.go +++ b/internal/hcl/modules/s3_cache.go @@ -63,7 +63,7 @@ func (cache *S3Cache) Exists(key string) (bool, error) { if err != nil { if awsErr, ok := err.(awserr.Error); ok { switch awsErr.Code() { - case s3.ErrCodeNoSuchKey, "NotFound": + case s3.ErrCodeNoSuchKey, "NotFound", "Forbidden": return false, nil } } From 67244a86f98a2361839ce96112252301a9f021e4 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Wed, 13 Nov 2024 22:20:48 +0000 Subject: [PATCH 41/81] enhance: update Terragrunt to use remote cache --- go.mod | 2 +- internal/hcl/modules/fetch.go | 50 +++++++++++------ internal/hcl/modules/fetch_test.go | 2 +- internal/hcl/modules/registry.go | 2 +- .../terraform/terragrunt_hcl_provider.go | 53 ++++++++++++------- 5 files changed, 71 insertions(+), 38 deletions(-) diff --git a/go.mod b/go.mod index 63abf281700..d4c7a485a4a 100644 --- a/go.mod +++ b/go.mod @@ -273,7 +273,7 @@ replace github.com/jedib0t/go-pretty/v6 => github.com/aliscott/go-pretty/v6 v6.1 replace github.com/spf13/cobra => github.com/spf13/cobra v1.4.0 -replace github.com/gruntwork-io/terragrunt => github.com/infracost/terragrunt v0.47.1-0.20240920091357-980f132ffa5a +replace github.com/gruntwork-io/terragrunt => github.com/infracost/terragrunt v0.47.1-0.20241113223044-d35392684463 replace github.com/heimdalr/dag => github.com/aliscott/dag v1.3.2-0.20231115114512-4ce18c825f94 diff --git a/internal/hcl/modules/fetch.go b/internal/hcl/modules/fetch.go index a36d94d5d62..51791d1d60c 100644 --- a/internal/hcl/modules/fetch.go +++ b/internal/hcl/modules/fetch.go @@ -10,7 +10,6 @@ import ( "sync" "time" - tgterraform "github.com/gruntwork-io/terragrunt/terraform" getter "github.com/hashicorp/go-getter" "github.com/otiai10/copy" "github.com/rs/zerolog" @@ -29,19 +28,50 @@ type PackageFetcher struct { localCache sync.Map remoteCache RemoteCache logger zerolog.Logger + getters map[string]getter.Getter } +type PackageFetcherOpts func(*PackageFetcher) + // NewPackageFetcher constructs a new package fetcher -func NewPackageFetcher(remoteCache RemoteCache, logger zerolog.Logger) *PackageFetcher { - return &PackageFetcher{ +func NewPackageFetcher(remoteCache RemoteCache, logger zerolog.Logger, opts ...PackageFetcherOpts) *PackageFetcher { + getters := make(map[string]getter.Getter, len(getter.Getters)) + for k, g := range getter.Getters { + getters[k] = g + } + getters["git"] = &CustomGitGetter{new(getter.GitGetter)} + + p := &PackageFetcher{ remoteCache: remoteCache, logger: logger, + getters: getters, + } + + for _, opt := range opts { + opt(p) + } + + return p +} + +func WithGetters(getters map[string]getter.Getter) PackageFetcherOpts { + return func(p *PackageFetcher) { + for k, g := range getters { + p.getters[k] = g + } } } // fetch downloads the remote module using the go-getter library // See: https://github.com/hashicorp/go-getter func (p *PackageFetcher) Fetch(moduleAddr string, dest string) error { + if strings.HasPrefix(moduleAddr, "file://") { + // Skip to the remote getter so it just copies this instead of + // looking up the cache + _, err := p.fetchFromRemote(moduleAddr, dest) + return err + } + fetched, err := p.fetchFromLocalCache(moduleAddr, dest) if fetched { p.logger.Trace().Msgf("cache hit (local): %s", moduleAddr) @@ -153,23 +183,13 @@ func (p *PackageFetcher) fetchFromRemote(moduleAddr, dest string) (bool, error) // I'm not sure if we really need it, but added it just in case/ decompressors["tar.tbz2"] = new(getter.TarBzip2Decompressor) - getters := make(map[string]getter.Getter, len(getter.Getters)) - for k, g := range getter.Getters { - getters[k] = g - } - - // This is a custom getter used by Terragrunt - getters["tfr"] = &tgterraform.RegistryGetter{} - - getters["git"] = &CustomGitGetter{new(getter.GitGetter)} - client := getter.Client{ Src: moduleAddr, Dst: dest, Pwd: dest, - Mode: getter.ClientModeDir, + Mode: getter.ClientModeAny, Decompressors: decompressors, - Getters: getters, + Getters: p.getters, } err := client.Get() diff --git a/internal/hcl/modules/fetch_test.go b/internal/hcl/modules/fetch_test.go index 22911c8d00b..eafa174e342 100644 --- a/internal/hcl/modules/fetch_test.go +++ b/internal/hcl/modules/fetch_test.go @@ -111,7 +111,7 @@ func TestPackageFetcher_fetch_RemoteCache(t *testing.T) { fetcher := NewPackageFetcher(mock, logger) - err = fetcher.fetch("git::https://github.com/terraform-aws-modules/terraform-aws-vpc?ref=v5.15.0", filepath.Join(tmpDir, "module")) + err = fetcher.Fetch("git::https://github.com/terraform-aws-modules/terraform-aws-vpc?ref=v5.15.0", filepath.Join(tmpDir, "module")) if tt.expectedError { assert.Error(t, err) return diff --git a/internal/hcl/modules/registry.go b/internal/hcl/modules/registry.go index fc7dc85635b..645fe549602 100644 --- a/internal/hcl/modules/registry.go +++ b/internal/hcl/modules/registry.go @@ -288,7 +288,7 @@ func (r *RegistryLoader) downloadModule(lookupResult *RegistryLookupResult, dest // Deliberately not logging the download URL since it contains a token r.logger.Debug().Msgf("Downloading module %s", lookupResult.ModuleURL.RawSource) - return r.packageFetcher.fetch(downloadURL, dest) + return r.packageFetcher.Fetch(downloadURL, dest) } func (r *RegistryLoader) DownloadLocation(moduleURL RegistryURL, version string) (string, error) { diff --git a/internal/providers/terraform/terragrunt_hcl_provider.go b/internal/providers/terraform/terragrunt_hcl_provider.go index c69c2ad25a7..f440b9a02bf 100644 --- a/internal/providers/terraform/terragrunt_hcl_provider.go +++ b/internal/providers/terraform/terragrunt_hcl_provider.go @@ -26,6 +26,7 @@ import ( "github.com/gruntwork-io/terragrunt/options" tgoptions "github.com/gruntwork-io/terragrunt/options" tfsource "github.com/gruntwork-io/terragrunt/terraform" + tgterraform "github.com/gruntwork-io/terragrunt/terraform" "github.com/gruntwork-io/terragrunt/util" "github.com/hashicorp/go-getter" hcl2 "github.com/hashicorp/hcl/v2" @@ -101,14 +102,14 @@ func (o *TerragruntOutputCache) Set(key string, getVal func() (cty.Value, error) } type TerragruntHCLProvider struct { - ctx *config.ProjectContext - Path hcl.RootPath - stack *tgconfigstack.Stack - excludedPaths []string - env map[string]string - sourceCache map[string]string - remoteCache modules.RemoteCache - logger zerolog.Logger + ctx *config.ProjectContext + Path hcl.RootPath + stack *tgconfigstack.Stack + excludedPaths []string + env map[string]string + sourceCache map[string]string + packageFetcher *modules.PackageFetcher + logger zerolog.Logger } // NewTerragruntHCLProvider creates a new provider initialized with the configured project path (usually the terragrunt @@ -129,14 +130,19 @@ func NewTerragruntHCLProvider(rootPath hcl.RootPath, ctx *config.ProjectContext) } } + fetcher := modules.NewPackageFetcher(remoteCache, logger, modules.WithGetters(map[string]getter.Getter{ + "tfr": &tgterraform.RegistryGetter{}, + "file": &tgcliterraform.FileCopyGetter{}, + })) + return &TerragruntHCLProvider{ - ctx: ctx, - Path: rootPath, - excludedPaths: ctx.ProjectConfig.ExcludePaths, - env: getEnvVars(ctx), - sourceCache: map[string]string{}, - remoteCache: remoteCache, - logger: logger, + ctx: ctx, + Path: rootPath, + excludedPaths: ctx.ProjectConfig.ExcludePaths, + env: getEnvVars(ctx), + sourceCache: map[string]string{}, + packageFetcher: fetcher, + logger: logger, } } @@ -439,6 +445,7 @@ func (p *TerragruntHCLProvider) prepWorkingDirs() ([]*terragruntWorkingDirInfo, Env: p.env, IgnoreExternalDependencies: true, SourceMap: p.ctx.RunContext.Config.TerraformSourceMap, + DownloadSource: p.downloadSource, UsePartialParseConfigCache: true, RunTerragrunt: func(opts *tgoptions.TerragruntOptions) (err error) { defer func() { @@ -577,7 +584,7 @@ func (p *TerragruntHCLProvider) runTerragrunt(opts *tgoptions.TerragruntOptions) return } if sourceURL != "" { - updatedWorkingDir, err := downloadSourceOnce(sourceURL, opts, terragruntConfig, p.remoteCache, p.logger) + updatedWorkingDir, err := loadSourceOnce(sourceURL, opts, terragruntConfig, p.packageFetcher, p.logger) if err != nil { info.error = err @@ -647,6 +654,12 @@ func (p *TerragruntHCLProvider) runTerragrunt(opts *tgoptions.TerragruntOptions) return info } +// downloadSource overrides the Terragrunt download source functionality to use the Infracost packageFetcher +// so that it makes use of any cached modules +func (p *TerragruntHCLProvider) downloadSource(downloadDir string, sourceURL string, opts *tgoptions.TerragruntOptions) error { + return p.packageFetcher.Fetch(sourceURL, downloadDir) +} + func splitModuleSubDir(moduleSource string) (string, string, error) { moduleAddr, submodulePath := getter.SourceDirSubdir(moduleSource) if strings.HasPrefix(submodulePath, "../") { @@ -656,8 +669,9 @@ func splitModuleSubDir(moduleSource string) (string, string, error) { return moduleAddr, submodulePath, nil } -// downloadSourceOnce thread-safely makes sure the sourceURL is only downloaded once -func downloadSourceOnce(sourceURL string, opts *tgoptions.TerragruntOptions, terragruntConfig *tgconfig.TerragruntConfig, remoteCache modules.RemoteCache, logger zerolog.Logger) (string, error) { +// loadSourceOnce thread-safely makes sure the sourceURL is only downloaded once. +// It calls the internal Terragrunt functionality to download the source. +func loadSourceOnce(sourceURL string, opts *tgoptions.TerragruntOptions, terragruntConfig *tgconfig.TerragruntConfig, packageFetcher *modules.PackageFetcher, logger zerolog.Logger) (string, error) { _, modAddr, err := splitModuleSubDir(sourceURL) if err != nil { return "", err @@ -717,8 +731,7 @@ func downloadSourceOnce(sourceURL string, opts *tgoptions.TerragruntOptions, ter logger.Trace().Msgf("recursively adding symlinked dirs to sparse-checkout for repo %s: %v", dir, symlinkedDirs) // Using a depth of 1 here since the submodule directory is already downloaded, so only need // to add the symlinked directories to the sparse-checkout. - packageFetcher := modules.NewPackageFetcher(remoteCache, logger) - err := modules.RecursivelyAddDirsToSparseCheckout(dir, parsedSourceURL, packageFetcher, []string{modAddr}, symlinkedDirs, mu, logger, 1) + err := modules.RecursivelyAddDirsToSparseCheckout(dir, sourceURL, packageFetcher, []string{modAddr}, symlinkedDirs, mu, logger, 1) if err != nil { return "", err } From 0b97cc5df6a12db9363a21aeb0fbdb6e6eaa3dea Mon Sep 17 00:00:00 2001 From: Tim McFadden <52185+tim775@users.noreply.github.com> Date: Fri, 15 Nov 2024 08:41:37 -0500 Subject: [PATCH 42/81] enhance: Add curent and past errors to project metadata (#3249) * enhance: Add curent and past errors to project metadata The project metadata previously contained a single errors field containing any errors that occurred in either side of a diff. This change makes it possible to determine which side of the diff the errors came from. * test: update golden files * test: update golden files dynamo prices dropped --- .../breakdown_with_actual_costs.golden | 6 +++--- ...own_with_actual_costs_pitr_disabled.golden | 6 +++--- ...with_current_and_past_project_error.golden | 16 +++++++++++++++ ...mpare_to_with_current_project_error.golden | 8 ++++++++ ..._compare_to_with_past_project_error.golden | 8 ++++++++ internal/output/combined.go | 3 +++ .../dynamodb_table_test.golden | 20 +++++++++---------- internal/schema/project.go | 4 +++- schema/infracost.schema.json | 12 +++++++++++ 9 files changed, 66 insertions(+), 17 deletions(-) diff --git a/cmd/infracost/testdata/breakdown_with_actual_costs/breakdown_with_actual_costs.golden b/cmd/infracost/testdata/breakdown_with_actual_costs/breakdown_with_actual_costs.golden index b8f15f2bcf8..e293f81d8c5 100644 --- a/cmd/infracost/testdata/breakdown_with_actual_costs/breakdown_with_actual_costs.golden +++ b/cmd/infracost/testdata/breakdown_with_actual_costs/breakdown_with_actual_costs.golden @@ -3,8 +3,8 @@ Project: main Name Monthly Qty Unit Monthly Cost aws_dynamodb_table.usage - ├─ Write request unit (WRU) 100,000 WRUs $0.13 * - ├─ Read request unit (RRU) 100,001 RRUs $0.03 * + ├─ Write request unit (WRU) 100,000 WRUs $0.06 * + ├─ Read request unit (RRU) 100,001 RRUs $0.01 * ├─ Data storage 100,002 GB $25,000.50 * ├─ Point-In-Time Recovery (PITR) backup storage 100,003 GB $20,000.60 * ├─ On-demand backup storage 100,004 GB $10,000.40 * @@ -16,7 +16,7 @@ Project: main └─ Actual costs Aug 15-Sep 22 (arn:another_aws_dynamodb_table) └─ $0.005123 per some aws thing 1,000 GB $5.12 - OVERALL TOTAL $70,002.42 + OVERALL TOTAL $70,002.35 *Usage costs were estimated using Infracost Cloud settings, see docs for other options. diff --git a/cmd/infracost/testdata/breakdown_with_actual_costs_pitr_disabled/breakdown_with_actual_costs_pitr_disabled.golden b/cmd/infracost/testdata/breakdown_with_actual_costs_pitr_disabled/breakdown_with_actual_costs_pitr_disabled.golden index 2f405debd40..146d71d6105 100644 --- a/cmd/infracost/testdata/breakdown_with_actual_costs_pitr_disabled/breakdown_with_actual_costs_pitr_disabled.golden +++ b/cmd/infracost/testdata/breakdown_with_actual_costs_pitr_disabled/breakdown_with_actual_costs_pitr_disabled.golden @@ -3,8 +3,8 @@ Project: main Name Monthly Qty Unit Monthly Cost aws_dynamodb_table.usage - ├─ Write request unit (WRU) 100,000 WRUs $0.13 * - ├─ Read request unit (RRU) 100,001 RRUs $0.03 * + ├─ Write request unit (WRU) 100,000 WRUs $0.06 * + ├─ Read request unit (RRU) 100,001 RRUs $0.01 * ├─ Data storage 100,002 GB $25,000.50 * ├─ On-demand backup storage 100,004 GB $10,000.40 * ├─ Table data restored 100,005 GB $15,000.75 * @@ -15,7 +15,7 @@ Project: main └─ Actual costs Aug 15-Sep 22 (arn:another_aws_dynamodb_table) └─ $0.005123 per some aws thing 1,000 GB $5.12 - OVERALL TOTAL $50,001.82 + OVERALL TOTAL $50,001.75 *Usage costs were estimated using Infracost Cloud settings, see docs for other options. diff --git a/cmd/infracost/testdata/diff_with_compare_to_with_current_and_past_project_error/diff_with_compare_to_with_current_and_past_project_error.golden b/cmd/infracost/testdata/diff_with_compare_to_with_current_and_past_project_error/diff_with_compare_to_with_current_and_past_project_error.golden index 09f1ed50a92..d052387b9fb 100644 --- a/cmd/infracost/testdata/diff_with_compare_to_with_current_and_past_project_error/diff_with_compare_to_with_current_and_past_project_error.golden +++ b/cmd/infracost/testdata/diff_with_compare_to_with_current_and_past_project_error/diff_with_compare_to_with_current_and_past_project_error.golden @@ -34,6 +34,22 @@ "data": null, "isError": false } + ], + "currentErrors": [ + { + "code": 102, + "message": "Error loading Terraform modules: failed to parse file REPLACED_PROJECT_PATH/testdata/diff_with_compare_to_with_current_and_past_project_error/dev/mod_with_error.tf diag: REPLACED_PROJECT_PATH/testdata/diff_with_compare_to_with_current_and_past_project_error/dev/mod_with_error.tf:1,9-10: Invalid block definition; Either a quoted string block label or an opening brace (\"{\") is expected here., and 1 other diagnostic(s)", + "data": null, + "isError": true + } + ], + "pastErrors": [ + { + "code": 102, + "message": "Diff baseline error: Error loading Terraform modules: failed to inspect module path dev diag: Invalid block definition: Either a quoted string block label or an opening brace (\"{\") is expected here. (and 1 other messages)", + "data": null, + "isError": false + } ] }, "pastBreakdown": { diff --git a/cmd/infracost/testdata/diff_with_compare_to_with_current_project_error/diff_with_compare_to_with_current_project_error.golden b/cmd/infracost/testdata/diff_with_compare_to_with_current_project_error/diff_with_compare_to_with_current_project_error.golden index 69b167571bc..ae2ba32bca4 100644 --- a/cmd/infracost/testdata/diff_with_compare_to_with_current_project_error/diff_with_compare_to_with_current_project_error.golden +++ b/cmd/infracost/testdata/diff_with_compare_to_with_current_project_error/diff_with_compare_to_with_current_project_error.golden @@ -28,6 +28,14 @@ "data": null, "isError": true } + ], + "currentErrors": [ + { + "code": 102, + "message": "Error loading Terraform modules: failed to parse file REPLACED_PROJECT_PATH/testdata/diff_with_compare_to_with_current_project_error/dev/mod_with_error.tf diag: REPLACED_PROJECT_PATH/testdata/diff_with_compare_to_with_current_project_error/dev/mod_with_error.tf:1,9-10: Invalid block definition; Either a quoted string block label or an opening brace (\"{\") is expected here., and 1 other diagnostic(s)", + "data": null, + "isError": true + } ] }, "pastBreakdown": { diff --git a/cmd/infracost/testdata/diff_with_compare_to_with_past_project_error/diff_with_compare_to_with_past_project_error.golden b/cmd/infracost/testdata/diff_with_compare_to_with_past_project_error/diff_with_compare_to_with_past_project_error.golden index 29bc0b6908c..b27e16cbd0b 100644 --- a/cmd/infracost/testdata/diff_with_compare_to_with_past_project_error/diff_with_compare_to_with_past_project_error.golden +++ b/cmd/infracost/testdata/diff_with_compare_to_with_past_project_error/diff_with_compare_to_with_past_project_error.golden @@ -29,6 +29,14 @@ "isError": false } ], + "pastErrors": [ + { + "code": 102, + "message": "Diff baseline error: Error loading Terraform modules: failed to inspect module path dev diag: Invalid block definition: Either a quoted string block label or an opening brace (\"{\") is expected here. (and 1 other messages)", + "data": null, + "isError": false + } + ], "providers": [ { "name": "aws", diff --git a/internal/output/combined.go b/internal/output/combined.go index 337edc8251c..578ed107832 100644 --- a/internal/output/combined.go +++ b/internal/output/combined.go @@ -138,6 +138,7 @@ func CompareTo(c *config.Config, current, prior Root) (Root, error) { scp.PastResources = nil scp.Metadata.PastPolicySha = "" scp.HasDiff = true + scp.Metadata.CurrentErrors = scp.Metadata.Errors metadata := p.LabelWithMetadata() if v, ok := priorProjects[metadata]; ok { @@ -165,6 +166,8 @@ func CompareTo(c *config.Config, current, prior Root) (Root, error) { continue } + scp.Metadata.PastErrors = append(scp.Metadata.PastErrors, pastE) + pastE.Message = "Diff baseline error: " + pastE.Message scp.Metadata.Errors = append(scp.Metadata.Errors, pastE) } diff --git a/internal/providers/terraform/aws/testdata/dynamodb_table_test/dynamodb_table_test.golden b/internal/providers/terraform/aws/testdata/dynamodb_table_test/dynamodb_table_test.golden index 92564079493..b22dc205b5a 100644 --- a/internal/providers/terraform/aws/testdata/dynamodb_table_test/dynamodb_table_test.golden +++ b/internal/providers/terraform/aws/testdata/dynamodb_table_test/dynamodb_table_test.golden @@ -2,17 +2,17 @@ Name Monthly Qty Unit Monthly Cost aws_dynamodb_table.my_dynamodb_table_usage - ├─ Write request unit (WRU) 3,000,000 WRUs $3.75 * - ├─ Read request unit (RRU) 8,000,000 RRUs $2.00 * + ├─ Write request unit (WRU) 3,000,000 WRUs $1.88 * + ├─ Read request unit (RRU) 8,000,000 RRUs $1.00 * ├─ Data storage 230 GB $57.50 * ├─ Point-In-Time Recovery (PITR) backup storage 2,300 GB $460.00 * ├─ On-demand backup storage 460 GB $46.00 * ├─ Table data restored 230 GB $34.50 * ├─ Streams read request unit (sRRU) 2,000,000 sRRUs $0.40 * ├─ Global table (us-east-2) - │ └─ Replicated write request unit (rWRU) 4,109.5890 rWRU $5.63 + │ └─ Replicated write request unit (rWRU) 4,109.5890 rWRU $1.88 └─ Global table (us-west-1) - └─ Replicated write request unit (rWRU) 4,109.5890 rWRU $6.27 + └─ Replicated write request unit (rWRU) 4,109.5890 rWRU $2.09 aws_dynamodb_table.autoscale_dynamodb_table_usage ├─ Write capacity unit (WCU, autoscaling) 77 WCU $36.54 * @@ -32,9 +32,9 @@ ├─ Table data restored Monthly cost depends on usage: $0.15 per GB ├─ Streams read request unit (sRRU) Monthly cost depends on usage: $0.0000002 per sRRUs ├─ Global table (us-east-2) - │ └─ Replicated write capacity unit (rWCU) 20 rWCU $14.24 + │ └─ Replicated write capacity unit (rWCU) 20 rWCU $9.49 └─ Global table (us-west-1) - └─ Replicated write capacity unit (rWCU) 20 rWCU $15.88 + └─ Replicated write capacity unit (rWCU) 20 rWCU $10.59 aws_dynamodb_table.my_dynamodb_table_with_no_billing_mode ├─ Write capacity unit (WCU) 20 WCU $9.49 @@ -45,9 +45,9 @@ ├─ Table data restored Monthly cost depends on usage: $0.15 per GB ├─ Streams read request unit (sRRU) Monthly cost depends on usage: $0.0000002 per sRRUs ├─ Global table (us-east-2) - │ └─ Replicated write capacity unit (rWCU) 20 rWCU $14.24 + │ └─ Replicated write capacity unit (rWCU) 20 rWCU $9.49 └─ Global table (us-west-1) - └─ Replicated write capacity unit (rWCU) 20 rWCU $15.88 + └─ Replicated write capacity unit (rWCU) 20 rWCU $10.59 aws_dynamodb_table.autoscale_dynamodb_table_literal_ref ├─ Write capacity unit (WCU) 20 WCU $9.49 @@ -67,7 +67,7 @@ ├─ Table data restored Monthly cost depends on usage: $0.15 per GB └─ Streams read request unit (sRRU) Monthly cost depends on usage: $0.0000002 per sRRUs - OVERALL TOTAL $762.82 + OVERALL TOTAL $731.93 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -78,5 +78,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $106 ┃ $657 ┃ $763 ┃ +┃ main ┃ $78 ┃ $654 ┃ $732 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/schema/project.go b/internal/schema/project.go index 080c00158ba..3ad2d3172fc 100644 --- a/internal/schema/project.go +++ b/internal/schema/project.go @@ -197,7 +197,9 @@ type ProjectMetadata struct { TerraformWorkspace string `json:"terraformWorkspace,omitempty"` VCSSubPath string `json:"vcsSubPath,omitempty"` VCSCodeChanged *bool `json:"vcsCodeChanged,omitempty"` - Errors []*ProjectDiag `json:"errors,omitempty"` + Errors []*ProjectDiag `json:"errors,omitempty"` // contains merged current and past errors + CurrentErrors []*ProjectDiag `json:"currentErrors,omitempty"` + PastErrors []*ProjectDiag `json:"pastErrors,omitempty"` Warnings []*ProjectDiag `json:"warnings,omitempty"` Policies Policies `json:"policies,omitempty"` Providers []ProviderMetadata `json:"providers,omitempty"` diff --git a/schema/infracost.schema.json b/schema/infracost.schema.json index 7380b8ba841..ecbabd6586e 100644 --- a/schema/infracost.schema.json +++ b/schema/infracost.schema.json @@ -355,6 +355,18 @@ }, "type": "array" }, + "currentErrors": { + "items": { + "$ref": "#/definitions/ProjectDiag" + }, + "type": "array" + }, + "pastErrors": { + "items": { + "$ref": "#/definitions/ProjectDiag" + }, + "type": "array" + }, "warnings": { "items": { "$ref": "#/definitions/ProjectDiag" From 114baef70209134d2714238aaf166b70fde9013c Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Sat, 16 Nov 2024 15:07:39 +0000 Subject: [PATCH 43/81] fix: don't check for symlinks for non-sparse checkout modules Only modules that have been sparse-checked out need this to resolve their symlinks properly --- internal/providers/terraform/terragrunt_hcl_provider.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/providers/terraform/terragrunt_hcl_provider.go b/internal/providers/terraform/terragrunt_hcl_provider.go index f440b9a02bf..f13976af6fa 100644 --- a/internal/providers/terraform/terragrunt_hcl_provider.go +++ b/internal/providers/terraform/terragrunt_hcl_provider.go @@ -682,9 +682,11 @@ func loadSourceOnce(sourceURL string, opts *tgoptions.TerragruntOptions, terragr return "", err } + sparseCheckoutEnabled := os.Getenv("INFRACOST_SPARSE_CHECKOUT") == "true" + // If sparse checkout is enabled add the subdir to the Source URL as a query param // so go-getter only downloads the required directory. - if os.Getenv("INFRACOST_SPARSE_CHECKOUT") == "true" { + if sparseCheckoutEnabled { q := parsedSourceURL.Query() q.Set("subdir", modAddr) parsedSourceURL.RawQuery = q.Encode() @@ -720,7 +722,7 @@ func loadSourceOnce(sourceURL string, opts *tgoptions.TerragruntOptions, terragr } } - if modAddr != "" && isGitDir(dir) { + if sparseCheckoutEnabled && modAddr != "" && isGitDir(dir) { symlinkedDirs, err := modules.ResolveSymLinkedDirs(dir, modAddr) if err != nil { return "", err From 568d78fa269b2fdda357b226afdd2a66bc7033a5 Mon Sep 17 00:00:00 2001 From: Tim McFadden <52185+tim775@users.noreply.github.com> Date: Tue, 19 Nov 2024 08:11:23 -0500 Subject: [PATCH 44/81] fix: cleanup bad module download directories (#3252) We need to clean up any partially downloaded modules if there is an error (e.g. if the module was downloaded but the ref couldn't be checked out), otherwise when another project tries to download the same module the `_, err = os.Stat(dest)` will decide that it has been previously downloaded successfully. --- internal/hcl/modules/loader.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/hcl/modules/loader.go b/internal/hcl/modules/loader.go index 2e7253aea9b..74d3a6b7627 100644 --- a/internal/hcl/modules/loader.go +++ b/internal/hcl/modules/loader.go @@ -801,6 +801,7 @@ func (m *ModuleLoader) loadRemoteModule(key string, source string) (*ManifestMod err = m.packageFetcher.Fetch(moduleAddr, dest) if err != nil { + _ = os.RemoveAll(dest) return nil, schema.NewFailedDownloadDiagnostic(source, err) } From 2ba293d0d72a539b5160c7c25efd9a0194736aa9 Mon Sep 17 00:00:00 2001 From: Liam Galvin Date: Tue, 19 Nov 2024 15:11:27 +0000 Subject: [PATCH 45/81] feat: Add option to create azure comments as active instead of defaulting to closed (#3253) * feat: Add option to create azure comments as active instead of defaulting to closed --- cmd/infracost/comment_azure_repos.go | 7 ++++-- .../comment_azure_repos_help.golden | 1 + .../completion_shell_bash.golden | 2 ++ internal/comment/azure_repos.go | 23 ++++++++++++------- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/cmd/infracost/comment_azure_repos.go b/cmd/infracost/comment_azure_repos.go index 6a7985f18f3..f06294431aa 100644 --- a/cmd/infracost/comment_azure_repos.go +++ b/cmd/infracost/comment_azure_repos.go @@ -41,9 +41,11 @@ func commentAzureReposCmd(ctx *config.RunContext) *cobra.Command { token, _ := cmd.Flags().GetString("azure-access-token") tag, _ := cmd.Flags().GetString("tag") + initActive, _ := cmd.Flags().GetBool("init-active") extra := comment.AzureReposExtra{ - Token: token, - Tag: tag, + Token: token, + Tag: tag, + InitActive: initActive, } prNumber, _ := cmd.Flags().GetInt("pull-request") @@ -149,6 +151,7 @@ func commentAzureReposCmd(ctx *config.RunContext) *cobra.Command { cmd.Flags().String("tag", "", "Customize hidden markdown tag used to detect comments posted by Infracost") cmd.Flags().Bool("dry-run", false, "Generate comment without actually posting to Azure Repos") cmd.Flags().String("format", "", "Output format: json") + cmd.Flags().Bool("init-active", false, "Initialize the comment as active instead of the default: closed") return cmd } diff --git a/cmd/infracost/testdata/comment_azure_repos_help/comment_azure_repos_help.golden b/cmd/infracost/testdata/comment_azure_repos_help/comment_azure_repos_help.golden index e92fb6dd446..40c4f57175f 100644 --- a/cmd/infracost/testdata/comment_azure_repos_help/comment_azure_repos_help.golden +++ b/cmd/infracost/testdata/comment_azure_repos_help/comment_azure_repos_help.golden @@ -17,6 +17,7 @@ FLAGS --dry-run Generate comment without actually posting to Azure Repos --format string Output format: json -h, --help help for azure-repos + --init-active Initialize the comment as active instead of the default: closed -p, --path stringArray Path to Infracost JSON files, glob patterns need quotes --policy-path stringArray Path to Infracost policy files, glob patterns need quotes (experimental) --pull-request int Pull request number to post comment on diff --git a/cmd/infracost/testdata/completion_shell_bash/completion_shell_bash.golden b/cmd/infracost/testdata/completion_shell_bash/completion_shell_bash.golden index 7ee4f085db9..296ebf5d81f 100644 --- a/cmd/infracost/testdata/completion_shell_bash/completion_shell_bash.golden +++ b/cmd/infracost/testdata/completion_shell_bash/completion_shell_bash.golden @@ -532,6 +532,8 @@ _infracost_comment_azure-repos() two_word_flags+=("--format") local_nonpersistent_flags+=("--format") local_nonpersistent_flags+=("--format=") + flags+=("--init-active") + local_nonpersistent_flags+=("--init-active") flags+=("--path=") two_word_flags+=("--path") flags_with_completion+=("--path") diff --git a/internal/comment/azure_repos.go b/internal/comment/azure_repos.go index 80f17dc4501..f1ef103a02c 100644 --- a/internal/comment/azure_repos.go +++ b/internal/comment/azure_repos.go @@ -66,7 +66,8 @@ type AzureReposExtra struct { // Token is the Azure DevOps access token. Token string // Tag is used to identify the Infracost comment. - Tag string + Tag string + InitActive bool } // azureAPIComment represents API response structure of Azure Repos comment. @@ -134,9 +135,10 @@ func buildAzureAPIURL(repoURL string) (string, error) { // implements the PlatformHandler interface and contains the functions // for finding, creating, updating, deleting comments on Azure Repos pull requests. type azureReposPRHandler struct { - httpClient *http.Client - repoAPIURL string - prNumber int + httpClient *http.Client + repoAPIURL string + prNumber int + initAsActive bool } // NewAzureReposPRHandler creates a new PlatformHandler for Azure Repos pull requests. @@ -157,9 +159,10 @@ func NewAzureReposPRHandler(ctx context.Context, repoURL string, targetRef strin } h := &azureReposPRHandler{ - httpClient: httpClient, - repoAPIURL: apiURL, - prNumber: prNumber, + httpClient: httpClient, + repoAPIURL: apiURL, + prNumber: prNumber, + initAsActive: extra.InitActive, } return NewCommentHandler(ctx, h, extra.Tag), nil @@ -236,6 +239,10 @@ func (h *azureReposPRHandler) CallFindMatchingComments(ctx context.Context, tag // CallCreateComment calls the Azure Repos API to create a new comment on the pull request. func (h *azureReposPRHandler) CallCreateComment(ctx context.Context, body string) (Comment, error) { + status := "closed" + if h.initAsActive { + status = "active" + } reqData, err := json.Marshal(map[string]interface{}{ "comments": []map[string]interface{}{ { @@ -244,7 +251,7 @@ func (h *azureReposPRHandler) CallCreateComment(ctx context.Context, body string "commentType": 1, }, }, - "status": 4, + "status": status, }) if err != nil { return nil, errors.Wrap(err, "Error marshaling comment body") From 08d91c880090c8e93dd6c22f7ccad10f9061ed14 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Wed, 20 Nov 2024 10:30:39 +0000 Subject: [PATCH 46/81] enhance: add parsing for git URLs to the source mapping --- go.mod | 1 + go.sum | 2 ++ internal/hcl/modules/loader.go | 13 +++++++++++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index d4c7a485a4a..962cbc113dc 100644 --- a/go.mod +++ b/go.mod @@ -216,6 +216,7 @@ require ( github.com/urfave/cli/v2 v2.27.2 // indirect github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + github.com/whilp/git-urls v1.0.0 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect diff --git a/go.sum b/go.sum index ca01b47eaea..7f1b9748dcc 100644 --- a/go.sum +++ b/go.sum @@ -1163,6 +1163,8 @@ github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU= +github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE= github.com/withfig/autocomplete-tools/packages/cobra v1.2.0 h1:MzD3XeOOSO3mAjOPpF07jFteSKZxsRHvlIcAR9RQzKM= github.com/withfig/autocomplete-tools/packages/cobra v1.2.0/go.mod h1:RoXh7+7qknOXL65uTzdzE1mPxqcPwS7FLCE9K5GfmKo= github.com/xanzy/go-gitlab v0.86.0 h1:jR8V9cK9jXRQDb46KOB20NCF3ksY09luaG0IfXE6p7w= diff --git a/internal/hcl/modules/loader.go b/internal/hcl/modules/loader.go index 74d3a6b7627..5d5076015d0 100644 --- a/internal/hcl/modules/loader.go +++ b/internal/hcl/modules/loader.go @@ -21,6 +21,7 @@ import ( "github.com/hashicorp/terraform-config-inspect/tfconfig" "github.com/otiai10/copy" "github.com/rs/zerolog" + giturls "github.com/whilp/git-urls" "golang.org/x/sync/errgroup" "github.com/infracost/infracost/internal/config" @@ -929,12 +930,20 @@ func mapSource(sourceMap config.TerraformSourceMap, source string) (SourceMapRes // Merge the query params from the source and dest URLs parsedSourceURL, err := url.Parse(moduleAddr) if err != nil { - return SourceMapResult{}, err + // Try parsing it as a git URL + parsedSourceURL, err = giturls.Parse(moduleAddr) + if err != nil { + return SourceMapResult{}, err + } } parsedDestURL, err := url.Parse(destSource) if err != nil { - return SourceMapResult{}, err + // Try parsing it as a git URL + parsedDestURL, err = giturls.Parse(destSource) + if err != nil { + return SourceMapResult{}, err + } } sourceQuery := parsedSourceURL.Query() From cf6e45e4c618319e7283a4a825242b96870d051c Mon Sep 17 00:00:00 2001 From: Liam Galvin Date: Thu, 21 Nov 2024 16:32:09 +0000 Subject: [PATCH 47/81] enhance: Add default timeout for retrieving remote modules (#3255) --- internal/hcl/modules/fetch.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/internal/hcl/modules/fetch.go b/internal/hcl/modules/fetch.go index 51791d1d60c..63610890929 100644 --- a/internal/hcl/modules/fetch.go +++ b/internal/hcl/modules/fetch.go @@ -1,6 +1,7 @@ package modules import ( + "context" "fmt" "net/url" "os" @@ -10,7 +11,7 @@ import ( "sync" "time" - getter "github.com/hashicorp/go-getter" + "github.com/hashicorp/go-getter" "github.com/otiai10/copy" "github.com/rs/zerolog" @@ -22,6 +23,8 @@ var commitRegex = regexp.MustCompile(`^([0-9a-f]{40})|([0-9a-f]{7})$`) var defaultTTL = 24 * time.Hour var tagCommitTTL = 30 * 24 * time.Hour +const defaultModuleRetrieveTimeout = 3 * time.Minute + // PackageFetcher downloads modules from a remote source to the given destination // This supports all the non-local and non-Terraform registry sources listed here: https://www.terraform.io/language/modules/sources type PackageFetcher struct { @@ -29,6 +32,7 @@ type PackageFetcher struct { remoteCache RemoteCache logger zerolog.Logger getters map[string]getter.Getter + timeout time.Duration } type PackageFetcherOpts func(*PackageFetcher) @@ -45,6 +49,7 @@ func NewPackageFetcher(remoteCache RemoteCache, logger zerolog.Logger, opts ...P remoteCache: remoteCache, logger: logger, getters: getters, + timeout: defaultModuleRetrieveTimeout, } for _, opt := range opts { @@ -62,6 +67,12 @@ func WithGetters(getters map[string]getter.Getter) PackageFetcherOpts { } } +var WithTimeout = func(d time.Duration) PackageFetcherOpts { + return func(p *PackageFetcher) { + p.timeout = d + } +} + // fetch downloads the remote module using the go-getter library // See: https://github.com/hashicorp/go-getter func (p *PackageFetcher) Fetch(moduleAddr string, dest string) error { @@ -183,7 +194,15 @@ func (p *PackageFetcher) fetchFromRemote(moduleAddr, dest string) (bool, error) // I'm not sure if we really need it, but added it just in case/ decompressors["tar.tbz2"] = new(getter.TarBzip2Decompressor) + ctx := context.Background() + if p.timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, p.timeout) + defer cancel() + } + client := getter.Client{ + Ctx: ctx, Src: moduleAddr, Dst: dest, Pwd: dest, From 36fce6f8b8b2ccddbd634e967a463065fedb369b Mon Sep 17 00:00:00 2001 From: Owen Date: Mon, 25 Nov 2024 14:46:53 +0000 Subject: [PATCH 48/81] fix: tidy up infracost dump path creation (#3258) Ensure that the dump location is created or fall back to the original path --- internal/providers/terraform/hcl_provider.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/internal/providers/terraform/hcl_provider.go b/internal/providers/terraform/hcl_provider.go index 82f4e9f2c5a..8051db27760 100644 --- a/internal/providers/terraform/hcl_provider.go +++ b/internal/providers/terraform/hcl_provider.go @@ -347,11 +347,16 @@ func (p *HCLProvider) LoadPlanJSON() HCLProject { module := p.Module() if module.Error == nil { module.JSON, module.Error = p.modulesToPlanJSON(module.Module) - if os.Getenv("INFRACOST_JSON_DUMP") == "true" { targetPath := fmt.Sprintf("%s-out.json", strings.ReplaceAll(module.Module.ModulePath, "/", "-")) - if os.Getenv("INFRACOST_JSON_DUMP_PATH") != "" { - targetPath = filepath.Join(os.Getenv("INFRACOST_JSON_DUMP_PATH"), targetPath) + targetDir, ok := os.LookupEnv("INFRACOST_JSON_DUMP_PATH") + if ok { + targetPath = filepath.Join(targetDir, targetPath) + if err := os.MkdirAll(targetDir, os.ModePerm); err != nil { + p.logger.Debug().Err(err).Msg("failed to create directory for json dump") + // use the default + targetPath = fmt.Sprintf("%s-out.json", strings.ReplaceAll(module.Module.ModulePath, "/", "-")) + } } err := os.WriteFile(targetPath, module.JSON, os.ModePerm) // nolint: gosec if err != nil { From 9804788656bf06be913b8bbe364e0c43adab89f4 Mon Sep 17 00:00:00 2001 From: Hugo Rut Date: Thu, 28 Nov 2024 13:17:11 +0100 Subject: [PATCH 49/81] enhance: output the response body when comment fails to post (#3259) Reads the body of the response if we get a non 200 code response from posting a comment. --- internal/comment/azure_repos.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/comment/azure_repos.go b/internal/comment/azure_repos.go index f1ef103a02c..d9c4f84421e 100644 --- a/internal/comment/azure_repos.go +++ b/internal/comment/azure_repos.go @@ -267,11 +267,16 @@ func (h *azureReposPRHandler) CallCreateComment(ctx context.Context, body string res, err := h.httpClient.Do(req) if err != nil { - return nil, errors.Wrap(err, "Error creating comment") + return nil, fmt.Errorf("Error creating comment: %w", err) } if res.StatusCode != http.StatusOK { - return nil, errors.Errorf("Error creating comment: %s", res.Status) + resBody, err := io.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("Error reading response body: %w", err) + } + + return nil, fmt.Errorf("Error creating comment: %s\n%s", res.Status, string(resBody)) } if res.Body != nil { From 45148551f4954df2006b9c4d04dada2149e1c8e3 Mon Sep 17 00:00:00 2001 From: Liam Galvin Date: Tue, 3 Dec 2024 14:19:45 +0000 Subject: [PATCH 50/81] fix: Handle and recover from go-cty panic (#3262) --- internal/providers/terraform/hcl_provider.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/internal/providers/terraform/hcl_provider.go b/internal/providers/terraform/hcl_provider.go index 8051db27760..c0d2309c9ec 100644 --- a/internal/providers/terraform/hcl_provider.go +++ b/internal/providers/terraform/hcl_provider.go @@ -695,6 +695,12 @@ func (p *HCLProvider) marshalAWSDefaultTagsBlock(providerBlock *hcl.Block) map[s return nil } + defer func() { + if r := recover(); r != nil { + p.logger.Debug().Msgf("could not marshal default_tags block: %v", r) + } + }() + marshalledTags := make(map[string]interface{}) tags := b.GetAttribute("tags") @@ -766,6 +772,12 @@ func (p *HCLProvider) marshalGoogleDefaultTagsBlock(providerBlock *hcl.Block) ma return nil } + defer func() { + if r := recover(); r != nil { + p.logger.Debug().Msgf("could not marshal default_labels block: %v", r) + } + }() + marshalledTags := make(map[string]interface{}) value := tags.Value() From e3bb8f5f8130e7405382a97ad44b52abf54017bd Mon Sep 17 00:00:00 2001 From: Tim McFadden <52185+tim775@users.noreply.github.com> Date: Wed, 4 Dec 2024 10:35:28 -0500 Subject: [PATCH 51/81] enhance: multiple price handling (#3263) * enhance: use the smallest non-zero price when multiple prices or products are found * enhance: ignore multiple prices if they're all the same * test: log multiple price/product errors as warnings when running tests This lets them get picked up as golden file diffs, implying a broken price lookup filter * fix: update prices filters for backup vault EFS backup * fix: fix Load balancer capacity units product filter * fix: fix lightsail cost filters * fix: restore CaptureLogs options and Fail on unexpected logs * test: capture expected logs for rds cluster tests * test: Add filter details to test log output on failed price lookups * fix: fix aurora mysql extended support lookup * fix: really fix load balancer capacity units lookup * fix: fix cognitive account speech to text filters * fix: fix virtual network public ip price filters * fix: fix google cloud run functions price filters * test: capture no price found logs since SNS -> SMS pricing is gone * test: disable terraform cli test for sfn state machine * fix: fix s3 intelligent tiering storage filters to work with cn regions * fix: azure global static public ip price filters --- cmd/infracost/run.go | 2 +- internal/prices/prices.go | 124 +++++++++++++----- .../aws/rds_cluster_instance_test.go | 5 +- .../terraform/aws/rds_cluster_test.go | 9 +- .../terraform/aws/sfn_state_machine_test.go | 10 +- .../providers/terraform/aws/sns_topic_test.go | 6 +- .../db_instance_test/db_instance_test.golden | 40 +++--- .../eks_cluster_test/eks_cluster_test.golden | 6 +- .../lightsail_instance_test.golden | 8 +- .../rds_cluster_china_test.golden | 6 +- .../rds_cluster_instance_test.golden | 18 +-- .../s3_bucket_china_test.golden | 8 +- .../sns_topic_test/sns_topic_test.golden | 9 +- .../providers/terraform/azure/public_ip.go | 11 +- .../application_gateway_test.golden | 6 +- .../cognitive_account_test.golden | 14 +- .../mssql_database_test.golden | 16 +-- .../mssql_managed_instance_test.golden | 8 +- .../public_ip_test/public_ip_test.golden | 10 +- .../sql_managed_instance_test.golden | 8 +- ...ual_network_gateway_connection_test.golden | 6 +- .../virtual_network_gateway_test.golden | 6 +- .../azure/virtual_network_gateway_test.go | 6 +- .../cloudfunctions_function_test.golden | 32 ++--- .../compute_address_test.golden | 6 +- .../compute_instance_test.golden | 14 +- .../compute_instance_test.tf | 24 ++-- .../container_cluster_test.golden | 18 +-- .../container_node_pool_test.golden | 12 +- internal/providers/terraform/tftest/tftest.go | 9 +- internal/resources/aws/backup_vault.go | 4 +- internal/resources/aws/db_instance.go | 18 ++- internal/resources/aws/lb.go | 2 +- internal/resources/aws/lightsail_instance.go | 42 +++--- .../s3_intelligent_tiering_storage_class.go | 4 +- .../azure/cognitive_account_speech.go | 4 +- .../google/cloudfunctions_function.go | 16 +-- internal/testutil/testutil.go | 9 ++ 38 files changed, 327 insertions(+), 229 deletions(-) diff --git a/cmd/infracost/run.go b/cmd/infracost/run.go index b05658efd7c..adb0260db49 100644 --- a/cmd/infracost/run.go +++ b/cmd/infracost/run.go @@ -243,7 +243,7 @@ func newParallelRunner(cmd *cobra.Command, runCtx *config.RunContext) (*parallel cmd: cmd, pathMuxs: pathMuxs, prior: prior, - pricingFetcher: prices.NewPriceFetcher(runCtx), + pricingFetcher: prices.NewPriceFetcher(runCtx, false), }, nil } diff --git a/internal/prices/prices.go b/internal/prices/prices.go index 59de42ac698..b267b625cca 100644 --- a/internal/prices/prices.go +++ b/internal/prices/prices.go @@ -17,7 +17,6 @@ import ( "github.com/infracost/infracost/internal/ui" "github.com/shopspring/decimal" - "github.com/tidwall/gjson" ) var ( @@ -35,20 +34,22 @@ type notFoundData struct { // data. This is used to provide a summary of missing prices at the end of a run. // It should be used as a singleton which is shared across the application. type PriceFetcher struct { - resources map[string]*notFoundData - components map[string]int - mux *sync.RWMutex - client *apiclient.PricingAPIClient - runCtx *config.RunContext + resources map[string]*notFoundData + components map[string]int + mux *sync.RWMutex + client *apiclient.PricingAPIClient + runCtx *config.RunContext + warnOnPriceErrors bool } -func NewPriceFetcher(ctx *config.RunContext) *PriceFetcher { +func NewPriceFetcher(ctx *config.RunContext, warnOnPriceErrors bool) *PriceFetcher { return &PriceFetcher{ - resources: make(map[string]*notFoundData), - components: make(map[string]int), - mux: &sync.RWMutex{}, - runCtx: ctx, - client: apiclient.NewPricingAPIClient(ctx), + resources: make(map[string]*notFoundData), + components: make(map[string]int), + mux: &sync.RWMutex{}, + runCtx: ctx, + client: apiclient.NewPricingAPIClient(ctx), + warnOnPriceErrors: warnOnPriceErrors, } } @@ -256,10 +257,25 @@ func (p *PriceFetcher) getPrices(req apiclient.BatchRequest) error { return nil } +func (p *PriceFetcher) logPriceLookupErr(cc *schema.CostComponent, format string, v ...interface{}) { + if p.warnOnPriceErrors { + productFilterJson, _ := json.Marshal(cc.ProductFilter) + priceFilterJson, _ := json.Marshal(cc.PriceFilter) + logging.Logger.Warn().Msgf(format+". Product filter: %s Price filter:%s", append(v, productFilterJson, priceFilterJson)...) + + } else { + logging.Logger.Debug().Msgf(format, v...) + } +} + +type productPrice struct { + Hash string + Price decimal.Decimal +} + func (p *PriceFetcher) setCostComponentPrice(result apiclient.PriceQueryResult) { currency := p.client.Currency - var pp decimal.Decimal if result.CostComponent.CustomPrice() != nil { logging.Logger.Debug().Msgf("Using user-defined custom price %v for %s %s.", *result.CostComponent.CustomPrice(), result.Resource.Name, result.CostComponent.Name) result.CostComponent.SetPrice(*result.CostComponent.CustomPrice()) @@ -273,6 +289,7 @@ func (p *PriceFetcher) setCostComponentPrice(result apiclient.PriceQueryResult) result.Resource.RemoveCostComponent(result.CostComponent) return } + p.logPriceLookupErr(result.CostComponent, "No products found for %s %s", result.Resource.Name, result.CostComponent.Name) p.addNotFoundResult(result) return @@ -284,44 +301,83 @@ func (p *PriceFetcher) setCostComponentPrice(result apiclient.PriceQueryResult) // Some resources may have identical records in CPAPI for the same product // filters, several products are always returned and they can only be - // distinguished by their prices. However if we pick the first product it may not + // distinguished by their prices. However, if we pick the first product it may not // have the price due to price filter and the lookup fails. Filtering the - // products with prices helps to solve that. - var productsWithPrices []gjson.Result + // products with prices helps to solve that. To make sure we get a consistent price + // between runs, we sort any multiple prices so we always pick the smallest non-zero + // price first. + var productPrices [][]productPrice + distinctPrices := map[string]bool{} for _, product := range products { - if len(product.Get("prices").Array()) > 0 { - productsWithPrices = append(productsWithPrices, product) + pricesResults := product.Get("prices").Array() + if len(pricesResults) > 0 { + // map pricesResults to decimals + var prices []productPrice + for _, price := range pricesResults { + priceStr := price.Get(currency).String() + p, err := decimal.NewFromString(priceStr) + if err != nil { + logging.Logger.Warn().Msgf("Error converting price to '%v' (using 0.00) '%v': %s", currency, price.Get(currency).String(), err.Error()) + prices = append(prices, productPrice{Hash: price.Get("priceHash").String(), Price: decimal.Zero}) + continue + } + prices = append(prices, productPrice{Hash: price.Get("priceHash").String(), Price: p}) + + distinctPrices[priceStr] = true + } + + // sort prices with the smallest non-zero price first + sort.Slice(prices, func(i, j int) bool { + if prices[i].Price.IsZero() { + return false // Treat zero as larger, so it goes to the end + } + if prices[j].Price.IsZero() { + return true // Non-zero should come before zero + } + // Both prices are non-zero, sort in ascending order + return prices[i].Price.LessThan(prices[j].Price) + }) + + productPrices = append(productPrices, prices) } } - if len(productsWithPrices) == 0 { + // sort productPrices with the smallest non-zero price first + sort.Slice(productPrices, func(i, j int) bool { + if productPrices[i][0].Price.IsZero() { + return false // Treat zero as larger, so it goes to the end + } + if productPrices[j][0].Price.IsZero() { + return true // Non-zero should come before zero + } + // Both prices are non-zero, sort in ascending order + return productPrices[i][0].Price.LessThan(productPrices[j][0].Price) + }) + + if len(productPrices) == 0 { if result.CostComponent.IgnoreIfMissingPrice { logging.Logger.Debug().Msgf("No prices found for %s %s, ignoring since IgnoreIfMissingPrice is set.", result.Resource.Name, result.CostComponent.Name) result.Resource.RemoveCostComponent(result.CostComponent) return } + p.logPriceLookupErr(result.CostComponent, "No prices found for %s %s", result.Resource.Name, result.CostComponent.Name) p.addNotFoundResult(result) return } - if len(productsWithPrices) > 1 { - logging.Logger.Debug().Msgf("Multiple products with prices found for %s %s, using the first product", result.Resource.Name, result.CostComponent.Name) - } + if len(distinctPrices) > 1 { + // only worry about duplicate products/prices if they have different prices. - prices := productsWithPrices[0].Get("prices").Array() - if len(prices) > 1 { - logging.Logger.Debug().Msgf("Multiple prices found for %s %s, using the first price", result.Resource.Name, result.CostComponent.Name) - } + if len(productPrices) > 1 { + p.logPriceLookupErr(result.CostComponent, "Multiple products with prices found for %s %s, using the smallest non-zero price", result.Resource.Name, result.CostComponent.Name) + } - var err error - pp, err = decimal.NewFromString(prices[0].Get(currency).String()) - if err != nil { - logging.Logger.Warn().Msgf("Error converting price to '%v' (using 0.00) '%v': %s", currency, prices[0].Get(currency).String(), err.Error()) - result.CostComponent.SetPrice(decimal.Zero) - return + if len(productPrices[0]) > 1 { + p.logPriceLookupErr(result.CostComponent, "Multiple prices found for %s %s, using the smallest non-zero price", result.Resource.Name, result.CostComponent.Name) + } } - result.CostComponent.SetPrice(pp) - result.CostComponent.SetPriceHash(prices[0].Get("priceHash").String()) + result.CostComponent.SetPrice(productPrices[0][0].Price) + result.CostComponent.SetPriceHash(productPrices[0][0].Hash) } diff --git a/internal/providers/terraform/aws/rds_cluster_instance_test.go b/internal/providers/terraform/aws/rds_cluster_instance_test.go index 406dffc04ce..0416c743216 100644 --- a/internal/providers/terraform/aws/rds_cluster_instance_test.go +++ b/internal/providers/terraform/aws/rds_cluster_instance_test.go @@ -7,10 +7,11 @@ import ( ) func TestRDSClusterInstanceGoldenFile(t *testing.T) { - t.Parallel() if testing.Short() { t.Skip("skipping test in short mode") } - tftest.GoldenFileResourceTests(t, "rds_cluster_instance_test") + tftest.GoldenFileResourceTestsWithOpts(t, "rds_cluster_instance_test", &tftest.GoldenFileOptions{ + CaptureLogs: true, + }) } diff --git a/internal/providers/terraform/aws/rds_cluster_test.go b/internal/providers/terraform/aws/rds_cluster_test.go index 25b37ad7c60..917f438c115 100644 --- a/internal/providers/terraform/aws/rds_cluster_test.go +++ b/internal/providers/terraform/aws/rds_cluster_test.go @@ -7,21 +7,22 @@ import ( ) func TestRDSClusterGoldenFile(t *testing.T) { - t.Parallel() if testing.Short() { t.Skip("skipping test in short mode") } - tftest.GoldenFileResourceTests(t, "rds_cluster_test") + tftest.GoldenFileResourceTestsWithOpts(t, "rds_cluster_test", &tftest.GoldenFileOptions{ + CaptureLogs: true, + }) } func TestRDSClusterChinaGoldenFile(t *testing.T) { - t.Parallel() if testing.Short() { t.Skip("skipping test in short mode") } tftest.GoldenFileResourceTestsWithOpts(t, "rds_cluster_china_test", &tftest.GoldenFileOptions{ - Currency: "CNY", + CaptureLogs: true, + Currency: "CNY", }) } diff --git a/internal/providers/terraform/aws/sfn_state_machine_test.go b/internal/providers/terraform/aws/sfn_state_machine_test.go index a60db16c667..6e77d46dbfb 100644 --- a/internal/providers/terraform/aws/sfn_state_machine_test.go +++ b/internal/providers/terraform/aws/sfn_state_machine_test.go @@ -12,5 +12,13 @@ func TestSFnStateMachineGoldenFile(t *testing.T) { t.Skip("skipping test in short mode") } - tftest.GoldenFileResourceTests(t, "sfn_state_machine_test") + // Don't run the terraform cli since it is complaining about the test examples: + // Error: validating Step Functions State Machine definition: + // operation error SFN: ValidateStateMachineDefinition, + // https response error StatusCode: 400, RequestID: 4059cfcd-96ae-4921-afb0-25a552d4f601, + // api error UnrecognizedClientException: The security token included in the request is invalid. + opts := tftest.DefaultGoldenFileOptions() + opts.IgnoreCLI = true + + tftest.GoldenFileHCLResourceTestsWithOpts(t, "sfn_state_machine_test", opts) } diff --git a/internal/providers/terraform/aws/sns_topic_test.go b/internal/providers/terraform/aws/sns_topic_test.go index d743e9c67ab..c5c8c4af7a8 100644 --- a/internal/providers/terraform/aws/sns_topic_test.go +++ b/internal/providers/terraform/aws/sns_topic_test.go @@ -7,10 +7,10 @@ import ( ) func TestSNSTopicGoldenFile(t *testing.T) { - t.Parallel() if testing.Short() { t.Skip("skipping test in short mode") } - - tftest.GoldenFileResourceTests(t, "sns_topic_test") + opts := tftest.DefaultGoldenFileOptions() + opts.CaptureLogs = true + tftest.GoldenFileResourceTestsWithOpts(t, "sns_topic_test", opts) } diff --git a/internal/providers/terraform/aws/testdata/db_instance_test/db_instance_test.golden b/internal/providers/terraform/aws/testdata/db_instance_test/db_instance_test.golden index ec7e0166744..86c887389de 100644 --- a/internal/providers/terraform/aws/testdata/db_instance_test/db_instance_test.golden +++ b/internal/providers/terraform/aws/testdata/db_instance_test/db_instance_test.golden @@ -41,6 +41,26 @@ ├─ Storage (general purpose SSD, gp2) 30 GB $6.90 └─ Additional backup storage 1,000 GB $95.00 * + aws_db_instance.extended_support["aurora-5.7"] + ├─ Database instance (on-demand, Single-AZ, db.t3.large) 730 hours $119.72 + ├─ Storage (general purpose SSD, gp2) 20 GB $2.30 + └─ Extended support (year 1) 1,460 vCPU-hours $146.00 + + aws_db_instance.extended_support["aurora-5.7.44"] + ├─ Database instance (on-demand, Single-AZ, db.t3.large) 730 hours $119.72 + ├─ Storage (general purpose SSD, gp2) 20 GB $2.30 + └─ Extended support (year 1) 1,460 vCPU-hours $146.00 + + aws_db_instance.extended_support["aurora-mysql-5.7"] + ├─ Database instance (on-demand, Single-AZ, db.t3.large) 730 hours $119.72 + ├─ Storage (general purpose SSD, gp2) 20 GB $2.30 + └─ Extended support (year 1) 1,460 vCPU-hours $146.00 + + aws_db_instance.extended_support["aurora-mysql-5.7.44"] + ├─ Database instance (on-demand, Single-AZ, db.t3.large) 730 hours $119.72 + ├─ Storage (general purpose SSD, gp2) 20 GB $2.30 + └─ Extended support (year 1) 1,460 vCPU-hours $146.00 + aws_db_instance.extended_support["aurora-postgresql-11"] ├─ Database instance (on-demand, Single-AZ, db.t3.large) 730 hours $119.72 ├─ Storage (general purpose SSD, gp2) 20 GB $2.30 @@ -126,14 +146,6 @@ ├─ Storage (general purpose SSD, gp2) 20 GB $2.30 └─ Additional backup storage Monthly cost depends on usage: $0.021 per GB - aws_db_instance.extended_support["aurora-5.7"] - ├─ Database instance (on-demand, Single-AZ, db.t3.large) 730 hours $119.72 - └─ Storage (general purpose SSD, gp2) 20 GB $2.30 - - aws_db_instance.extended_support["aurora-5.7.44"] - ├─ Database instance (on-demand, Single-AZ, db.t3.large) 730 hours $119.72 - └─ Storage (general purpose SSD, gp2) 20 GB $2.30 - aws_db_instance.extended_support["aurora-8.0"] ├─ Database instance (on-demand, Single-AZ, db.t3.large) 730 hours $119.72 └─ Storage (general purpose SSD, gp2) 20 GB $2.30 @@ -142,14 +154,6 @@ ├─ Database instance (on-demand, Single-AZ, db.t3.large) 730 hours $119.72 └─ Storage (general purpose SSD, gp2) 20 GB $2.30 - aws_db_instance.extended_support["aurora-mysql-5.7"] - ├─ Database instance (on-demand, Single-AZ, db.t3.large) 730 hours $119.72 - └─ Storage (general purpose SSD, gp2) 20 GB $2.30 - - aws_db_instance.extended_support["aurora-mysql-5.7.44"] - ├─ Database instance (on-demand, Single-AZ, db.t3.large) 730 hours $119.72 - └─ Storage (general purpose SSD, gp2) 20 GB $2.30 - aws_db_instance.extended_support["aurora-mysql-8.0"] ├─ Database instance (on-demand, Single-AZ, db.t3.large) 730 hours $119.72 └─ Storage (general purpose SSD, gp2) 20 GB $2.30 @@ -306,7 +310,7 @@ ├─ Database instance (reserved, Single-AZ, db.t3.large) 730 hours $0.00 └─ Storage (general purpose SSD, gp2) 20 GB $2.30 - OVERALL TOTAL $11,880.39 + OVERALL TOTAL $12,464.39 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -317,5 +321,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $10,719 ┃ $1,161 ┃ $11,880 ┃ +┃ main ┃ $11,303 ┃ $1,161 ┃ $12,464 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/aws/testdata/eks_cluster_test/eks_cluster_test.golden b/internal/providers/terraform/aws/testdata/eks_cluster_test/eks_cluster_test.golden index c5eebf46947..d1a21769ac1 100644 --- a/internal/providers/terraform/aws/testdata/eks_cluster_test/eks_cluster_test.golden +++ b/internal/providers/terraform/aws/testdata/eks_cluster_test/eks_cluster_test.golden @@ -17,7 +17,7 @@ └─ EKS cluster (extended support) 730 hours $438.00 aws_eks_cluster.extended_support["1.28"] - └─ EKS cluster 730 hours $73.00 + └─ EKS cluster (extended support) 730 hours $438.00 aws_eks_cluster.extended_support["1.29"] └─ EKS cluster 730 hours $73.00 @@ -25,7 +25,7 @@ aws_eks_cluster.my_aws_eks_cluster └─ EKS cluster 730 hours $73.00 - OVERALL TOTAL $2,409.00 + OVERALL TOTAL $2,774.00 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -36,5 +36,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $2,409 ┃ - ┃ $2,409 ┃ +┃ main ┃ $2,774 ┃ - ┃ $2,774 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/aws/testdata/lightsail_instance_test/lightsail_instance_test.golden b/internal/providers/terraform/aws/testdata/lightsail_instance_test/lightsail_instance_test.golden index 64ac985370d..207ca5ec482 100644 --- a/internal/providers/terraform/aws/testdata/lightsail_instance_test/lightsail_instance_test.golden +++ b/internal/providers/terraform/aws/testdata/lightsail_instance_test/lightsail_instance_test.golden @@ -2,12 +2,12 @@ Name Monthly Qty Unit Monthly Cost aws_lightsail_instance.linux1 - └─ Virtual server (Linux/UNIX) 730 hours $78.49 + └─ Virtual server (Linux/UNIX) 730 hours $82.42 aws_lightsail_instance.win1 - └─ Virtual server (Windows) 730 hours $19.62 + └─ Virtual server (Windows) 730 hours $21.58 - OVERALL TOTAL $98.11 + OVERALL TOTAL $104.00 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -18,5 +18,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $98 ┃ - ┃ $98 ┃ +┃ main ┃ $104 ┃ - ┃ $104 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/aws/testdata/rds_cluster_china_test/rds_cluster_china_test.golden b/internal/providers/terraform/aws/testdata/rds_cluster_china_test/rds_cluster_china_test.golden index df0b060a248..7288856ef37 100644 --- a/internal/providers/terraform/aws/testdata/rds_cluster_china_test/rds_cluster_china_test.golden +++ b/internal/providers/terraform/aws/testdata/rds_cluster_china_test/rds_cluster_china_test.golden @@ -19,4 +19,8 @@ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ ┃ main ┃ 0.00 元 ┃ - ┃ 0.00 元 ┃ -┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file +┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ +Logs: + +WARN No products found for aws_rds_cluster.mysql_china Backup storage. Product filter: {"vendorName":"aws","service":"AmazonRDS","productFamily":"Storage Snapshot","region":"cn-north-1","attributeFilters":[{"key":"databaseEngine","value_regex":"/Aurora MySQL/i"},{"key":"usagetype","value_regex":"/Aurora:BackupUsage$/i"}]} Price filter:null +WARN No products found for aws_rds_cluster.mysql_china Snapshot export. Product filter: {"vendorName":"aws","service":"AmazonRDS","productFamily":"System Operation","region":"cn-north-1","attributeFilters":[{"key":"databaseEngine","value_regex":"/Aurora MySQL/i"},{"key":"usagetype","value_regex":"/Aurora:SnapshotExportToS3$/i"}]} Price filter:null \ No newline at end of file diff --git a/internal/providers/terraform/aws/testdata/rds_cluster_instance_test/rds_cluster_instance_test.golden b/internal/providers/terraform/aws/testdata/rds_cluster_instance_test/rds_cluster_instance_test.golden index c52ed2c7f49..28b43368802 100644 --- a/internal/providers/terraform/aws/testdata/rds_cluster_instance_test/rds_cluster_instance_test.golden +++ b/internal/providers/terraform/aws/testdata/rds_cluster_instance_test/rds_cluster_instance_test.golden @@ -14,6 +14,14 @@ aws_rds_cluster_instance.aurora_serverless_v2_with_usage └─ Aurora serverless v2 3,650 ACU-hours $438.00 * + aws_rds_cluster_instance.extended_support["aurora-mysql-5.7"] + ├─ Database instance (on-demand, db.t3.large) 730 hours $119.72 + └─ Extended support (year 1) 1,460 vCPU-hours $146.00 + + aws_rds_cluster_instance.extended_support["aurora-mysql-5.7.44"] + ├─ Database instance (on-demand, db.t3.large) 730 hours $119.72 + └─ Extended support (year 1) 1,460 vCPU-hours $146.00 + aws_rds_cluster_instance.extended_support["aurora-postgresql-11"] ├─ Database instance (on-demand, db.t3.large) 730 hours $119.72 └─ Extended support (year 1) 1,460 vCPU-hours $146.00 @@ -34,12 +42,6 @@ ├─ Performance Insights Long Term Retention (db.t3.large) 2 vCPU-month $5.90 └─ Performance Insights API 12.345 1000 requests $0.12 * - aws_rds_cluster_instance.extended_support["aurora-mysql-5.7"] - └─ Database instance (on-demand, db.t3.large) 730 hours $119.72 - - aws_rds_cluster_instance.extended_support["aurora-mysql-5.7.44"] - └─ Database instance (on-demand, db.t3.large) 730 hours $119.72 - aws_rds_cluster_instance.extended_support["aurora-mysql-8.0"] └─ Database instance (on-demand, db.t3.large) 730 hours $119.72 @@ -105,7 +107,7 @@ ├─ Aurora serverless v2 Monthly cost depends on usage: $0.12 per ACU-hours └─ Performance Insights API Monthly cost depends on usage: $0.01 per 1000 requests - OVERALL TOTAL $6,695.11 + OVERALL TOTAL $6,987.11 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -116,5 +118,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $5,809 ┃ $886 ┃ $6,695 ┃ +┃ main ┃ $6,101 ┃ $886 ┃ $6,987 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/aws/testdata/s3_bucket_china_test/s3_bucket_china_test.golden b/internal/providers/terraform/aws/testdata/s3_bucket_china_test/s3_bucket_china_test.golden index 2c5975a6547..81a58302236 100644 --- a/internal/providers/terraform/aws/testdata/s3_bucket_china_test/s3_bucket_china_test.golden +++ b/internal/providers/terraform/aws/testdata/s3_bucket_china_test/s3_bucket_china_test.golden @@ -11,8 +11,8 @@ ├─ Intelligent tiering │ ├─ Storage (frequent access) 20,000 GB 3,900.00 元 * │ ├─ Storage (infrequent access) 20,000 GB 2,675.40 元 * - │ ├─ Storage (archive access) 20,000 GB not found * - │ ├─ Storage (deep archive access) 20,000 GB not found * + │ ├─ Storage (archive access) 20,000 GB 601.20 元 * + │ ├─ Storage (deep archive access) 20,000 GB 267.20 元 * │ ├─ Monitoring and automation 20 1k objects 0.33 元 * │ ├─ PUT, COPY, POST, LIST requests 20 1k requests 0.09 元 * │ ├─ GET, SELECT, and all other requests 20 1k requests 0.03 元 * @@ -71,7 +71,7 @@ ├─ Select data scanned Monthly cost depends on usage: 0.01593 元 per GB └─ Select data returned Monthly cost depends on usage: 0.0057 元 per GB - OVERALL TOTAL (CNY) 87,161.77 元 + OVERALL TOTAL (CNY) 88,030.17 元 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -82,5 +82,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ 0.00 元 ┃ 87,162 元 ┃ 87,162 元 ┃ +┃ main ┃ 0.00 元 ┃ 88,030 元 ┃ 88,030 元 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/aws/testdata/sns_topic_test/sns_topic_test.golden b/internal/providers/terraform/aws/testdata/sns_topic_test/sns_topic_test.golden index 0d589e9a713..6792810250f 100644 --- a/internal/providers/terraform/aws/testdata/sns_topic_test/sns_topic_test.golden +++ b/internal/providers/terraform/aws/testdata/sns_topic_test/sns_topic_test.golden @@ -76,4 +76,11 @@ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ ┃ main ┃ $0.00 ┃ $27 ┃ $27 ┃ -┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file +┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ +Logs: + +WARN No prices found for aws_sns_topic.sns_topic SMS notifications (over 100). Product filter: {"vendorName":"aws","service":"AmazonSNS","productFamily":"Message Delivery","region":"us-east-1","attributeFilters":[{"key":"usagetype","value_regex":"/DeliveryAttempts-SMS$/i"}]} Price filter:{"startUsageAmount":"100"} +WARN No prices found for aws_sns_topic.sns_topic_another_region SMS notifications (over 100). Product filter: {"vendorName":"aws","service":"AmazonSNS","productFamily":"Message Delivery","region":"us-east-1","attributeFilters":[{"key":"usagetype","value_regex":"/DeliveryAttempts-SMS$/i"}]} Price filter:{"startUsageAmount":"100"} +WARN No prices found for aws_sns_topic.sns_topic_withChargedSubscribers SMS notifications (over 100). Product filter: {"vendorName":"aws","service":"AmazonSNS","productFamily":"Message Delivery","region":"us-east-1","attributeFilters":[{"key":"usagetype","value_regex":"/DeliveryAttempts-SMS$/i"}]} Price filter:{"startUsageAmount":"100"} +WARN No prices found for aws_sns_topic.sns_topic_withFreeNotifications SMS notifications (over 100). Product filter: {"vendorName":"aws","service":"AmazonSNS","productFamily":"Message Delivery","region":"us-east-1","attributeFilters":[{"key":"usagetype","value_regex":"/DeliveryAttempts-SMS$/i"}]} Price filter:{"startUsageAmount":"100"} +WARN No prices found for aws_sns_topic.sns_topic_withZeroRequests SMS notifications (over 100). Product filter: {"vendorName":"aws","service":"AmazonSNS","productFamily":"Message Delivery","region":"us-east-1","attributeFilters":[{"key":"usagetype","value_regex":"/DeliveryAttempts-SMS$/i"}]} Price filter:{"startUsageAmount":"100"} \ No newline at end of file diff --git a/internal/providers/terraform/azure/public_ip.go b/internal/providers/terraform/azure/public_ip.go index e606da05306..b60730e290f 100644 --- a/internal/providers/terraform/azure/public_ip.go +++ b/internal/providers/terraform/azure/public_ip.go @@ -23,12 +23,17 @@ func NewAzureRMPublicIP(d *schema.ResourceData, u *schema.UsageData) *schema.Res var meterName string sku := "Standard" // default sku is Standard skuTier := "Regional" - allocationMethod := d.Get("allocation_method").String() if d.Get("sku").Type != gjson.Null { sku = d.Get("sku").String() } + allocationMethod := strings.ToLower(d.Get("allocation_method").String()) + if allocationMethod == "dynamic" { + // dynamic IPs imply a basic sku. + sku = "Basic" + } + switch sku { case "Basic": meterName = "Basic IPv4 " + allocationMethod + " Public IP" @@ -38,7 +43,7 @@ func NewAzureRMPublicIP(d *schema.ResourceData, u *schema.UsageData) *schema.Res skuTier = skuTierVal } if skuTier == "Global" { - sku = "Standard" // When sku_tier is Global, sku is Standard + sku = "Global" // When sku_tier is Global, skuname is global meterName = "Global IPv4 " + allocationMethod + " Public IP" } else { meterName = "Standard IPv4 " + allocationMethod + " Public IP" @@ -70,7 +75,7 @@ func PublicIPCostComponent(name, region, sku, meterName string) *schema.CostComp AttributeFilters: []*schema.AttributeFilter{ {Key: "productName", Value: strPtr("IP Addresses")}, {Key: "skuName", Value: strPtr(sku)}, - {Key: "meterName", Value: strPtr(meterName)}, + {Key: "meterName", ValueRegex: regexPtr(meterName)}, }, }, PriceFilter: &schema.PriceFilter{ diff --git a/internal/providers/terraform/azure/testdata/application_gateway_test/application_gateway_test.golden b/internal/providers/terraform/azure/testdata/application_gateway_test/application_gateway_test.golden index da18e697d91..c048384c135 100644 --- a/internal/providers/terraform/azure/testdata/application_gateway_test/application_gateway_test.golden +++ b/internal/providers/terraform/azure/testdata/application_gateway_test/application_gateway_test.golden @@ -42,9 +42,9 @@ └─ Data processing (0-10TB) Monthly cost depends on usage: $0.00 per GB azurerm_public_ip.example - └─ IP address (dynamic, regional) 730 hours not found + └─ IP address (dynamic, regional) 730 hours $2.92 - OVERALL TOTAL $5,911.62 + OVERALL TOTAL $5,914.54 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -56,5 +56,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $3,571 ┃ $2,340 ┃ $5,912 ┃ +┃ main ┃ $3,574 ┃ $2,340 ┃ $5,915 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/cognitive_account_test/cognitive_account_test.golden b/internal/providers/terraform/azure/testdata/cognitive_account_test/cognitive_account_test.golden index 11e45b1303f..d8cc110f351 100644 --- a/internal/providers/terraform/azure/testdata/cognitive_account_test/cognitive_account_test.golden +++ b/internal/providers/terraform/azure/testdata/cognitive_account_test/cognitive_account_test.golden @@ -178,9 +178,9 @@ └─ Text analytics for health (5K-500K) Monthly cost depends on usage: $25.00 per 1K records azurerm_cognitive_account.with_usage["SpeechServices-S0"] - ├─ Speech to text 20 hours not found + ├─ Speech to text 20 hours $20.00 ├─ Speech to text batch 56 hours $10.08 - ├─ Speech to text custom model 17 hours not found + ├─ Speech to text custom model 17 hours $20.40 ├─ Speech to text custom model batch 44 hours $9.90 ├─ Speech to text custom endpoint hosting 372 hours $20.00 ├─ Speech to text custom training 2 hours $20.00 @@ -218,7 +218,7 @@ azurerm_cognitive_account.speech_with_commitment["invalid"] ├─ Speech to text batch Monthly cost depends on usage: $0.18 per hours - ├─ Speech to text custom model not found + ├─ Speech to text custom model Monthly cost depends on usage: $1.20 per hours ├─ Speech to text custom model batch Monthly cost depends on usage: $0.23 per hours ├─ Speech to text custom endpoint hosting Monthly cost depends on usage: $0.05375 per hours ├─ Speech to text custom training Monthly cost depends on usage: $10.00 per hours @@ -250,9 +250,9 @@ └─ Speech requests Monthly cost depends on usage: $5.50 per 1K transactions azurerm_cognitive_account.without_usage["SpeechServices-S0"] - ├─ Speech to text not found + ├─ Speech to text Monthly cost depends on usage: $1.00 per hours ├─ Speech to text batch Monthly cost depends on usage: $0.18 per hours - ├─ Speech to text custom model not found + ├─ Speech to text custom model Monthly cost depends on usage: $1.20 per hours ├─ Speech to text custom model batch Monthly cost depends on usage: $0.23 per hours ├─ Speech to text custom endpoint hosting Monthly cost depends on usage: $0.05375 per hours ├─ Speech to text custom training Monthly cost depends on usage: $10.00 per hours @@ -281,7 +281,7 @@ ├─ Customized training Monthly cost depends on usage: $3.00 per hour └─ Text analytics for health (5K-500K) Monthly cost depends on usage: $25.00 per 1K records - OVERALL TOTAL $314,836.35 + OVERALL TOTAL $314,876.75 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -295,7 +295,7 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $314,836 ┃ - ┃ $314,836 ┃ +┃ main ┃ $314,877 ┃ - ┃ $314,877 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ Logs: diff --git a/internal/providers/terraform/azure/testdata/mssql_database_test/mssql_database_test.golden b/internal/providers/terraform/azure/testdata/mssql_database_test/mssql_database_test.golden index 6c0472fd532..7408f50cae4 100644 --- a/internal/providers/terraform/azure/testdata/mssql_database_test/mssql_database_test.golden +++ b/internal/providers/terraform/azure/testdata/mssql_database_test/mssql_database_test.golden @@ -55,19 +55,19 @@ ├─ Long-term retention (RA-GRS) 1,000 GB $50.00 * └─ PITR backup storage (RA-GRS) 500 GB $100.00 * - azurerm_mssql_database.backup_zone + azurerm_mssql_database.backup_local ├─ Compute (provisioned, GP_Gen5_4) 730 hours $444.47 ├─ SQL license 2,920 vCore-hours $291.90 ├─ Storage 5 GB $0.58 - ├─ Long-term retention (ZRS) 1,000 GB $31.30 * - └─ PITR backup storage (ZRS) 500 GB $62.50 * + ├─ Long-term retention (LRS) 1,000 GB $25.00 * + └─ PITR backup storage (LRS) 500 GB $50.00 * - azurerm_mssql_database.backup_local + azurerm_mssql_database.backup_zone ├─ Compute (provisioned, GP_Gen5_4) 730 hours $444.47 ├─ SQL license 2,920 vCore-hours $291.90 ├─ Storage 5 GB $0.58 - ├─ Long-term retention (LRS) 1,000 GB $25.00 * - └─ PITR backup storage (LRS) 500 GB $50.00 * + ├─ Long-term retention (ZRS) 1,000 GB $25.00 * + └─ PITR backup storage (ZRS) 500 GB $50.00 * azurerm_mssql_database.general_purpose_gen ├─ Compute (provisioned, GP_Gen5_4) 730 hours $444.47 @@ -116,7 +116,7 @@ ├─ Long-term retention (RA-GRS) Monthly cost depends on usage: $0.05 per GB └─ PITR backup storage (RA-GRS) Monthly cost depends on usage: $0.20 per GB - OVERALL TOTAL $31,585.37 + OVERALL TOTAL $31,566.57 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -128,5 +128,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $30,593 ┃ $992 ┃ $31,585 ┃ +┃ main ┃ $30,593 ┃ $973 ┃ $31,567 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/mssql_managed_instance_test/mssql_managed_instance_test.golden b/internal/providers/terraform/azure/testdata/mssql_managed_instance_test/mssql_managed_instance_test.golden index 8816050244e..0b1f002db4f 100644 --- a/internal/providers/terraform/azure/testdata/mssql_managed_instance_test/mssql_managed_instance_test.golden +++ b/internal/providers/terraform/azure/testdata/mssql_managed_instance_test/mssql_managed_instance_test.golden @@ -4,8 +4,8 @@ azurerm_mssql_managed_instance.example3 ├─ Compute (BC_GEN5 40 cores) 730 hours $9,778.47 ├─ Additional Storage 32 GB $4.38 - ├─ PITR backup storage (ZRS) 100 GB $14.90 * - └─ LTR backup storage (ZRS) 100 GB $3.72 * + ├─ PITR backup storage (ZRS) 100 GB $11.90 * + └─ LTR backup storage (ZRS) 100 GB $2.98 * azurerm_mssql_managed_instance.example2 ├─ Compute (GP_GEN5 16 cores) 730 hours $1,955.69 @@ -20,7 +20,7 @@ ├─ SQL license 2,920 vCore-hours $291.90 └─ LTR backup storage (LRS) 5 GB $0.15 * - OVERALL TOTAL $12,558.78 + OVERALL TOTAL $12,555.04 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -32,5 +32,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $12,528 ┃ $31 ┃ $12,559 ┃ +┃ main ┃ $12,528 ┃ $27 ┃ $12,555 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/public_ip_test/public_ip_test.golden b/internal/providers/terraform/azure/testdata/public_ip_test/public_ip_test.golden index 8c1fc396f32..99cfa0fef79 100644 --- a/internal/providers/terraform/azure/testdata/public_ip_test/public_ip_test.golden +++ b/internal/providers/terraform/azure/testdata/public_ip_test/public_ip_test.golden @@ -1,16 +1,16 @@ Name Monthly Qty Unit Monthly Cost + azurerm_public_ip.global + └─ IP address (static, global) 730 hours $7.30 + azurerm_public_ip.example └─ IP address (static, regional) 730 hours $3.65 azurerm_public_ip.example1 └─ IP address (dynamic, regional) 730 hours $2.92 - azurerm_public_ip.global - └─ IP address (static, global) 730 hours not found - - OVERALL TOTAL $6.57 + OVERALL TOTAL $13.87 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -22,5 +22,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $7 ┃ - ┃ $7 ┃ +┃ main ┃ $14 ┃ - ┃ $14 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/sql_managed_instance_test/sql_managed_instance_test.golden b/internal/providers/terraform/azure/testdata/sql_managed_instance_test/sql_managed_instance_test.golden index 25518db5c22..ce7acf993e5 100644 --- a/internal/providers/terraform/azure/testdata/sql_managed_instance_test/sql_managed_instance_test.golden +++ b/internal/providers/terraform/azure/testdata/sql_managed_instance_test/sql_managed_instance_test.golden @@ -4,8 +4,8 @@ azurerm_sql_managed_instance.example3 ├─ Compute (BC_GEN5 40 Cores) 730 hours $9,778.47 ├─ Storage 32 GB $4.38 - ├─ PITR backup storage (ZRS) 100 GB $14.90 * - └─ LTR backup storage (ZRS) 100 GB $3.72 * + ├─ PITR backup storage (ZRS) 100 GB $11.90 * + └─ LTR backup storage (ZRS) 100 GB $2.98 * azurerm_sql_managed_instance.example2 ├─ Compute (GP_GEN5 16 Cores) 730 hours $1,955.69 @@ -20,7 +20,7 @@ ├─ SQL license 2,920 vCore-hours $291.90 └─ LTR backup storage (LRS) 5 GB $0.15 * - OVERALL TOTAL $12,558.78 + OVERALL TOTAL $12,555.04 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -32,5 +32,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $12,528 ┃ $31 ┃ $12,559 ┃ +┃ main ┃ $12,528 ┃ $27 ┃ $12,555 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/virtual_network_gateway_connection_test/virtual_network_gateway_connection_test.golden b/internal/providers/terraform/azure/testdata/virtual_network_gateway_connection_test/virtual_network_gateway_connection_test.golden index 01cd28254bd..53ee88be933 100644 --- a/internal/providers/terraform/azure/testdata/virtual_network_gateway_connection_test/virtual_network_gateway_connection_test.golden +++ b/internal/providers/terraform/azure/testdata/virtual_network_gateway_connection_test/virtual_network_gateway_connection_test.golden @@ -55,9 +55,9 @@ └─ VPN gateway (VpnGw5) 730 hours $10.95 azurerm_public_ip.example - └─ IP address (dynamic, regional) 730 hours not found + └─ IP address (dynamic, regional) 730 hours $2.92 - OVERALL TOTAL $5,961.91 + OVERALL TOTAL $5,964.83 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -69,5 +69,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $5,962 ┃ - ┃ $5,962 ┃ +┃ main ┃ $5,965 ┃ - ┃ $5,965 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/testdata/virtual_network_gateway_test/virtual_network_gateway_test.golden b/internal/providers/terraform/azure/testdata/virtual_network_gateway_test/virtual_network_gateway_test.golden index f1f1924fa69..b37532eb3f4 100644 --- a/internal/providers/terraform/azure/testdata/virtual_network_gateway_test/virtual_network_gateway_test.golden +++ b/internal/providers/terraform/azure/testdata/virtual_network_gateway_test/virtual_network_gateway_test.golden @@ -37,9 +37,9 @@ └─ VPN gateway data tranfer Monthly cost depends on usage: $0.035 per GB azurerm_public_ip.example - └─ IP address (dynamic, regional) 730 hours not found + └─ IP address (dynamic, regional) 730 hours $2.92 - OVERALL TOTAL $49,244.35 + OVERALL TOTAL $49,247.27 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -51,5 +51,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $49,244 ┃ - ┃ $49,244 ┃ +┃ main ┃ $49,247 ┃ - ┃ $49,247 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/azure/virtual_network_gateway_test.go b/internal/providers/terraform/azure/virtual_network_gateway_test.go index 078bf04aae1..34f6c363fd5 100644 --- a/internal/providers/terraform/azure/virtual_network_gateway_test.go +++ b/internal/providers/terraform/azure/virtual_network_gateway_test.go @@ -7,11 +7,9 @@ import ( ) func TestAzureRMVirtualNetworkGateway(t *testing.T) { + t.Parallel() if testing.Short() { t.Skip("skipping test in short mode") } - - opts := tftest.DefaultGoldenFileOptions() - opts.CaptureLogs = true - tftest.GoldenFileResourceTestsWithOpts(t, "virtual_network_gateway_test", opts) + tftest.GoldenFileResourceTests(t, "virtual_network_gateway_test") } diff --git a/internal/providers/terraform/google/testdata/cloudfunctions_function_test/cloudfunctions_function_test.golden b/internal/providers/terraform/google/testdata/cloudfunctions_function_test/cloudfunctions_function_test.golden index a4e134b0b16..ff155e2e473 100644 --- a/internal/providers/terraform/google/testdata/cloudfunctions_function_test/cloudfunctions_function_test.golden +++ b/internal/providers/terraform/google/testdata/cloudfunctions_function_test/cloudfunctions_function_test.golden @@ -1,19 +1,19 @@ - Name Monthly Qty Unit Monthly Cost - - google_cloudfunctions_function.my_function - ├─ CPU 1,200,000 GHz-seconds not found * - ├─ Memory 750,000 GB-seconds not found * - ├─ Invocations 10,000,000 invocations not found * - └─ Outbound data transfer 100 GB not found * - - google_cloudfunctions_function.function - ├─ CPU not found - ├─ Memory not found - ├─ Invocations not found - └─ Outbound data transfer not found - - OVERALL TOTAL $0.00 + Name Monthly Qty Unit Monthly Cost + + google_cloudfunctions_function.my_function + ├─ CPU 1,200,000 GHz-seconds $12.00 * + ├─ Memory 750,000 GB-seconds $1.88 * + ├─ Invocations 10,000,000 invocations $4.00 * + └─ Outbound data transfer 100 GB $12.00 * + + google_cloudfunctions_function.function + ├─ CPU Monthly cost depends on usage: $0.00001 per GHz-seconds + ├─ Memory Monthly cost depends on usage: $0.0000025 per GB-seconds + ├─ Invocations Monthly cost depends on usage: $0.0000004 per invocations + └─ Outbound data transfer Monthly cost depends on usage: $0.12 per GB + + OVERALL TOTAL $29.88 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -24,5 +24,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $0.00 ┃ - ┃ $0.00 ┃ +┃ main ┃ $0.00 ┃ $30 ┃ $30 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/google/testdata/compute_address_test/compute_address_test.golden b/internal/providers/terraform/google/testdata/compute_address_test/compute_address_test.golden index 9f9897c59d4..ab24065393f 100644 --- a/internal/providers/terraform/google/testdata/compute_address_test/compute_address_test.golden +++ b/internal/providers/terraform/google/testdata/compute_address_test/compute_address_test.golden @@ -25,13 +25,13 @@ └─ IP address (standard VM) 730 hours $3.65 google_compute_instance.preemptible_instance_with_ip - ├─ Instance usage (Linux/UNIX, preemptible, f1-micro) 730 hours $2.56 + ├─ Instance usage (Linux/UNIX, preemptible, f1-micro) 730 hours $2.22 └─ Standard provisioned storage (pd-standard) 10 GB $0.40 google_compute_address.preemptible_static └─ IP address (preemptible VM) 730 hours $1.83 - OVERALL TOTAL $47.66 + OVERALL TOTAL $47.32 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -43,5 +43,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $48 ┃ - ┃ $48 ┃ +┃ main ┃ $47 ┃ - ┃ $47 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/google/testdata/compute_instance_test/compute_instance_test.golden b/internal/providers/terraform/google/testdata/compute_instance_test/compute_instance_test.golden index 9ef4a6cef26..147ee653640 100644 --- a/internal/providers/terraform/google/testdata/compute_instance_test/compute_instance_test.golden +++ b/internal/providers/terraform/google/testdata/compute_instance_test/compute_instance_test.golden @@ -16,7 +16,7 @@ └─ NVIDIA L4 (on-demand) 730 hours $286.18 google_compute_instance.preemptible_gpu - ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 730 hours $152.19 + ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 730 hours $144.60 ├─ Standard provisioned storage (pd-standard) 10 GB $0.40 └─ NVIDIA Tesla K80 (preemptible) 2,920 hours $501.95 @@ -56,8 +56,8 @@ └─ Local SSD provisioned storage 750 GB $60.00 google_compute_instance.custom_preemptible - ├─ Custom instance CPU (Linux/UNIX, preemptible, N1 6 vCPUs) 730 hours $40.47 - ├─ Custom Instance RAM (Linux/UNIX, preemptible, N1 20 GB) 730 hours $17.29 + ├─ Custom instance CPU (Linux/UNIX, preemptible, N1 6 vCPUs) 730 hours $38.46 + ├─ Custom Instance RAM (Linux/UNIX, preemptible, N1 20 GB) 730 hours $16.43 └─ Standard provisioned storage (pd-standard) 10 GB $0.40 google_compute_instance.custom_n2d @@ -66,7 +66,7 @@ └─ Standard provisioned storage (pd-standard) 10 GB $0.40 google_compute_instance.preemptible_local_ssd - ├─ Instance usage (Linux/UNIX, preemptible, f1-micro) 730 hours $2.56 + ├─ Instance usage (Linux/UNIX, preemptible, f1-micro) 730 hours $2.22 ├─ Standard provisioned storage (pd-standard) 10 GB $0.40 └─ Local SSD provisioned storage 375 GB $21.00 @@ -79,14 +79,14 @@ └─ Standard provisioned storage (pd-standard) 10 GB $0.40 google_compute_instance.preemptible - ├─ Instance usage (Linux/UNIX, preemptible, f1-micro) 730 hours $2.56 + ├─ Instance usage (Linux/UNIX, preemptible, f1-micro) 730 hours $2.22 └─ Standard provisioned storage (pd-standard) 10 GB $0.40 google_compute_instance.with_hours ├─ Instance usage (Linux/UNIX, on-demand, f1-micro) 100 hours $0.76 └─ Standard provisioned storage (pd-standard) 10 GB $0.40 - OVERALL TOTAL $10,356.12 + OVERALL TOTAL $10,344.98 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -99,7 +99,7 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $10,356 ┃ - ┃ $10,356 ┃ +┃ main ┃ $10,345 ┃ - ┃ $10,345 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ Logs: diff --git a/internal/providers/terraform/google/testdata/compute_instance_test/compute_instance_test.tf b/internal/providers/terraform/google/testdata/compute_instance_test/compute_instance_test.tf index 821856b2f0a..a53022e9617 100644 --- a/internal/providers/terraform/google/testdata/compute_instance_test/compute_instance_test.tf +++ b/internal/providers/terraform/google/testdata/compute_instance_test/compute_instance_test.tf @@ -59,7 +59,7 @@ resource "google_compute_instance" "preemptible" { } resource "google_compute_instance" "local_ssd" { - name = "local_ssd" + name = "local-ssd" machine_type = "f1-micro" zone = "us-central1-a" @@ -83,7 +83,7 @@ resource "google_compute_instance" "local_ssd" { } resource "google_compute_instance" "preemptible_local_ssd" { - name = "preemptible_local_ssd" + name = "preemptible-local-ssd" machine_type = "f1-micro" zone = "us-central1-a" @@ -128,7 +128,7 @@ resource "google_compute_instance" "gpu" { } resource "google_compute_instance" "gpu_l4" { - name = "gpu_l4" + name = "gpu-l4" machine_type = "g2-standard-4" zone = "us-central1-a" @@ -149,7 +149,7 @@ resource "google_compute_instance" "gpu_l4" { } resource "google_compute_instance" "preemptible_gpu" { - name = "preemptible_gpu" + name = "preemptible-gpu" machine_type = "n1-standard-16" zone = "us-central1-a" @@ -227,7 +227,7 @@ resource "google_compute_instance" "custom" { } resource "google_compute_instance" "custom_preemptible" { - name = "custom_preemptible" + name = "custom-preemptible" machine_type = "custom-6-20480" zone = "us-central1-a" @@ -247,7 +247,7 @@ resource "google_compute_instance" "custom_preemptible" { } resource "google_compute_instance" "custom_n1" { - name = "custom_n1" + name = "custom-n1" machine_type = "n1-custom-6-20480" zone = "us-central1-a" @@ -263,7 +263,7 @@ resource "google_compute_instance" "custom_n1" { } resource "google_compute_instance" "custom_n2" { - name = "custom_n2" + name = "custom-n2" machine_type = "n2-custom-6-20480" zone = "us-central1-a" @@ -280,7 +280,7 @@ resource "google_compute_instance" "custom_n2" { resource "google_compute_instance" "custom_n2d" { - name = "custom_n2d" + name = "custom-n2d" machine_type = "n2d-custom-4-20480" zone = "us-central1-a" @@ -300,7 +300,7 @@ resource "google_compute_instance" "custom_n2d" { } resource "google_compute_instance" "custom_ext" { - name = "custom_ext" + name = "custom-ext" machine_type = "custom-2-15360-ext" zone = "us-central1-a" @@ -317,7 +317,7 @@ resource "google_compute_instance" "custom_ext" { // Not supported yet resource "google_compute_instance" "e2_custom" { - name = "e2_custom" + name = "e2-custom" machine_type = "e2-custom-2-15360" zone = "us-central1-a" @@ -333,7 +333,7 @@ resource "google_compute_instance" "e2_custom" { } resource "google_compute_instance" "sud_20_perc_with_hours" { - name = "n2_standard_8" + name = "n2-standard-8" machine_type = "n2-standard-8" zone = "us-central1-a" @@ -349,7 +349,7 @@ resource "google_compute_instance" "sud_20_perc_with_hours" { } resource "google_compute_instance" "sud_30_perc_with_hours" { - name = "m1_ultramem_80" + name = "m1-ultramem-80" machine_type = "m1-ultramem-80" zone = "us-central1-a" diff --git a/internal/providers/terraform/google/testdata/container_cluster_test/container_cluster_test.golden b/internal/providers/terraform/google/testdata/container_cluster_test/container_cluster_test.golden index 27a21705262..f09b2d1842b 100644 --- a/internal/providers/terraform/google/testdata/container_cluster_test/container_cluster_test.golden +++ b/internal/providers/terraform/google/testdata/container_cluster_test/container_cluster_test.golden @@ -10,7 +10,7 @@ │ ├─ Instance usage (Linux/UNIX, on-demand, n1-standard-16) 8,760 hours $4,660.30 │ └─ Standard provisioned storage (pd-standard) 1,200 GB $48.00 └─ node_pool[1] - ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 8,760 hours $1,826.28 + ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 8,760 hours $1,735.18 └─ Standard provisioned storage (pd-standard) 1,200 GB $48.00 google_container_cluster.with_node_pools_regional @@ -22,7 +22,7 @@ │ ├─ Instance usage (Linux/UNIX, on-demand, n1-standard-16) 4,380 hours $2,330.15 │ └─ Standard provisioned storage (pd-standard) 600 GB $24.00 └─ node_pool[1] - ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 8,760 hours $1,826.28 + ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 8,760 hours $1,735.18 └─ Standard provisioned storage (pd-standard) 1,200 GB $48.00 google_container_cluster.with_node_config @@ -42,7 +42,7 @@ │ ├─ Instance usage (Linux/UNIX, on-demand, n1-standard-16) 5,840 hours $3,106.86 │ └─ Standard provisioned storage (pd-standard) 800 GB $32.00 └─ node_pool[1] - ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 2,920 hours $608.76 + ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 2,920 hours $578.39 └─ Standard provisioned storage (pd-standard) 400 GB $16.00 google_container_cluster.with_node_pools_zonal_withUsage @@ -54,7 +54,7 @@ │ ├─ Instance usage (Linux/UNIX, on-demand, n1-standard-16) 2,920 hours $1,553.43 │ └─ Standard provisioned storage (pd-standard) 400 GB $16.00 └─ node_pool[1] - ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 2,920 hours $608.76 + ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 2,920 hours $578.39 └─ Standard provisioned storage (pd-standard) 400 GB $16.00 google_container_cluster.with_unsupported_node_pool @@ -63,7 +63,7 @@ │ ├─ Instance usage (Linux/UNIX, on-demand, e2-medium) 6,570 hours $220.13 │ └─ Standard provisioned storage (pd-standard) 900 GB $36.00 └─ node_pool[1] - ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 8,760 hours $1,826.28 + ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 8,760 hours $1,735.18 └─ Standard provisioned storage (pd-standard) 1,200 GB $48.00 google_container_cluster.with_node_pools_node_locations @@ -75,7 +75,7 @@ │ ├─ Instance usage (Linux/UNIX, on-demand, n1-standard-16) 2,920 hours $1,553.43 │ └─ Standard provisioned storage (pd-standard) 400 GB $16.00 └─ node_pool[1] - ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 1,460 hours $304.38 + ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 1,460 hours $289.20 └─ Standard provisioned storage (pd-standard) 200 GB $8.00 google_container_cluster.with_node_pools_zonal @@ -87,7 +87,7 @@ │ ├─ Instance usage (Linux/UNIX, on-demand, n1-standard-16) 1,460 hours $776.72 │ └─ Standard provisioned storage (pd-standard) 200 GB $8.00 └─ node_pool[1] - ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 2,920 hours $608.76 + ├─ Instance usage (Linux/UNIX, preemptible, n1-standard-16) 2,920 hours $578.39 └─ Standard provisioned storage (pd-standard) 400 GB $16.00 google_container_cluster.autopilot_with_usage @@ -138,7 +138,7 @@ ├─ Autopilot memory Monthly cost depends on usage: $3.59 per GB └─ Autopilot ephemeral storage Monthly cost depends on usage: $0.040004 per GB - OVERALL TOTAL $30,162.55 + OVERALL TOTAL $29,782.95 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -149,7 +149,7 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $29,654 ┃ $509 ┃ $30,163 ┃ +┃ main ┃ $29,274 ┃ $509 ┃ $29,783 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ Logs: diff --git a/internal/providers/terraform/google/testdata/container_node_pool_test/container_node_pool_test.golden b/internal/providers/terraform/google/testdata/container_node_pool_test/container_node_pool_test.golden index 3093f0d666e..432cd39ad66 100644 --- a/internal/providers/terraform/google/testdata/container_node_pool_test/container_node_pool_test.golden +++ b/internal/providers/terraform/google/testdata/container_node_pool_test/container_node_pool_test.golden @@ -65,13 +65,13 @@ └─ Standard provisioned storage (pd-standard) 800 GB $32.00 google_container_node_pool.with_preemptible_instance - ├─ Custom instance CPU (Linux/UNIX, preemptible, N1 6 vCPUs) 2,190 hours $121.41 - ├─ Custom Instance RAM (Linux/UNIX, preemptible, N1 20 GB) 2,190 hours $51.86 + ├─ Custom instance CPU (Linux/UNIX, preemptible, N1 6 vCPUs) 2,190 hours $115.37 + ├─ Custom Instance RAM (Linux/UNIX, preemptible, N1 20 GB) 2,190 hours $49.28 └─ Standard provisioned storage (pd-standard) 300 GB $12.00 google_container_node_pool.with_spot_instance - ├─ Custom instance CPU (Linux/UNIX, preemptible, N1 6 vCPUs) 2,190 hours $121.41 - ├─ Custom Instance RAM (Linux/UNIX, preemptible, N1 20 GB) 2,190 hours $51.86 + ├─ Custom instance CPU (Linux/UNIX, preemptible, N1 6 vCPUs) 2,190 hours $115.37 + ├─ Custom Instance RAM (Linux/UNIX, preemptible, N1 20 GB) 2,190 hours $49.28 └─ Standard provisioned storage (pd-standard) 300 GB $12.00 google_container_node_pool.autoscaling_zonal @@ -122,7 +122,7 @@ ├─ Instance usage (Linux/UNIX, on-demand, e2-medium) 1,460 hours $48.92 └─ Standard provisioned storage (pd-standard) 200 GB $8.00 - OVERALL TOTAL $28,076.01 + OVERALL TOTAL $28,058.75 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -133,5 +133,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ $28,076 ┃ - ┃ $28,076 ┃ +┃ main ┃ $28,059 ┃ - ┃ $28,059 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/tftest/tftest.go b/internal/providers/terraform/tftest/tftest.go index e05dc33c6c6..50c9d6498b8 100644 --- a/internal/providers/terraform/tftest/tftest.go +++ b/internal/providers/terraform/tftest/tftest.go @@ -250,7 +250,12 @@ func goldenFileResourceTestWithOpts(t *testing.T, pName string, testName string, level = *options.LogLevel } - logBuf := testutil.ConfigureTestToCaptureLogs(t, runCtx, level) + var logBuf *bytes.Buffer + if options != nil && options.CaptureLogs { + logBuf = testutil.ConfigureTestToCaptureLogs(t, runCtx, level) + } else { + testutil.ConfigureTestToFailOnLogs(t, runCtx) + } if options != nil && options.Currency != "" { runCtx.Config.Currency = options.Currency @@ -349,7 +354,7 @@ func loadResources(t *testing.T, pName string, tfProject TerraformProject, runCt } func RunCostCalculations(runCtx *config.RunContext, projects []*schema.Project) ([]*schema.Project, error) { - pf := prices.NewPriceFetcher(runCtx) + pf := prices.NewPriceFetcher(runCtx, true) for _, project := range projects { err := pf.PopulatePrices(project) if err != nil { diff --git a/internal/resources/aws/backup_vault.go b/internal/resources/aws/backup_vault.go index 25b69843800..6d7c8abbe1a 100644 --- a/internal/resources/aws/backup_vault.go +++ b/internal/resources/aws/backup_vault.go @@ -54,13 +54,13 @@ func (r *BackupVault) PopulateUsage(u *schema.UsageData) { func (r *BackupVault) BuildResource() *schema.Resource { costComponents := []*schema.CostComponent{} - bd := backupData{ref: "monthly_efs_warm_backup_gb", name: "EFS backup (warm)", unit: "GB", usageType: "WarmStorage-ByteHrs-EFS", service: "AWSBackup", family: "AWS Backup Storage"} + bd := backupData{ref: "monthly_efs_warm_backup_gb", name: "EFS backup (warm)", unit: "GB", usageType: "WarmStorage-ByteHrs-EFS$", service: "AWSBackup", family: "AWS Backup Storage"} if r.MonthlyEFSWarmBackupGB != nil { bd.qty = decimalPtr(decimal.NewFromFloat(*r.MonthlyEFSWarmBackupGB)) } costComponents = append(costComponents, r.backupVaultCostComponent(bd)) - bd = backupData{ref: "monthly_efs_cold_backup_gb", name: "EFS backup (cold)", unit: "GB", usageType: "EarlyDelete-ColdByteHrs-EFS", service: "AWSBackup", family: "AWS Backup Early Delete Size"} + bd = backupData{ref: "monthly_efs_cold_backup_gb", name: "EFS backup (cold)", unit: "GB", usageType: "ColdStorage-ByteHrs-EFS$", service: "AWSBackup", family: "AWS Backup Storage"} if r.MonthlyEFSColdBackupGB != nil { bd.qty = decimalPtr(decimal.NewFromFloat(*r.MonthlyEFSColdBackupGB)) } diff --git a/internal/resources/aws/db_instance.go b/internal/resources/aws/db_instance.go index f50eab36689..c49e483905d 100644 --- a/internal/resources/aws/db_instance.go +++ b/internal/resources/aws/db_instance.go @@ -471,8 +471,9 @@ func performanceInsightsAPIRequestCostComponent(region string, additionalRequest // engine version Year1 is the date when the extended support starts, Year 3 is // the date when the extended increases price. type ExtendedSupportDates struct { - Year1 time.Time - Year3 time.Time + UsagetypeVersion string + Year1 time.Time + Year3 time.Time } // ExtendedSupport contains the extended support dates for a specific RDS engine. @@ -506,6 +507,11 @@ func (s ExtendedSupport) CostComponent(version string, region string, d time.Tim } } + usagetypeVersion := supportDates.UsagetypeVersion + if usagetypeVersion == "" { + usagetypeVersion = matchingVersion + } + if !supportDates.Year3.IsZero() && d.After(supportDates.Year3) { return &schema.CostComponent{ Name: "Extended support (year 3)", @@ -517,7 +523,7 @@ func (s ExtendedSupport) CostComponent(version string, region string, d time.Tim Region: strPtr(region), Service: strPtr("AmazonRDS"), AttributeFilters: []*schema.AttributeFilter{ - {Key: "usagetype", ValueRegex: regexPtr("ExtendedSupport:Yr3:" + s.Engine + matchingVersion)}, + {Key: "usagetype", ValueRegex: regexPtr("ExtendedSupport:Yr3:" + s.Engine + usagetypeVersion)}, }, }, } @@ -534,7 +540,7 @@ func (s ExtendedSupport) CostComponent(version string, region string, d time.Tim Region: strPtr(region), Service: strPtr("AmazonRDS"), AttributeFilters: []*schema.AttributeFilter{ - {Key: "usagetype", ValueRegex: regexPtr("ExtendedSupport:Yr1-Yr2:" + s.Engine + matchingVersion)}, + {Key: "usagetype", ValueRegex: regexPtr("ExtendedSupport:Yr1-Yr2:" + s.Engine + usagetypeVersion)}, }, }, } @@ -557,8 +563,8 @@ var ( mysqlAuroraExtendedSupport = ExtendedSupport{ Engine: "AuroraMySQL", Versions: map[string]ExtendedSupportDates{ - "5.7": {Year1: time.Date(2024, time.December, 1, 0, 0, 0, 0, time.UTC)}, // Year3 is zero because it's N/A - "8": {Year1: time.Date(2027, time.May, 1, 0, 0, 0, 0, time.UTC)}, // Year3 is zero because it's N/A + "5.7": {UsagetypeVersion: "2", Year1: time.Date(2024, time.December, 1, 0, 0, 0, 0, time.UTC)}, // Year3 is zero because it's N/A + "8": {UsagetypeVersion: "3", Year1: time.Date(2027, time.May, 1, 0, 0, 0, 0, time.UTC)}, // Year3 is zero because it's N/A }, } diff --git a/internal/resources/aws/lb.go b/internal/resources/aws/lb.go index ded4175e2d1..64549f2b68a 100644 --- a/internal/resources/aws/lb.go +++ b/internal/resources/aws/lb.go @@ -145,7 +145,7 @@ func (r *LB) capacityUnitsCostComponent(productFamily string, maxLCU *decimal.De ProductFamily: strPtr(productFamily), AttributeFilters: []*schema.AttributeFilter{ {Key: "locationType", Value: strPtr("AWS Region")}, - {Key: "usagetype", ValueRegex: strPtr("/LCUUsage/")}, + {Key: "usagetype", ValueRegex: strPtr("/^([A-Z]{3}\\d-|Global-|EU-)?LCUUsage/")}, }, }, UsageBased: true, diff --git a/internal/resources/aws/lightsail_instance.go b/internal/resources/aws/lightsail_instance.go index 11a947a7bb6..c917d5ecb62 100644 --- a/internal/resources/aws/lightsail_instance.go +++ b/internal/resources/aws/lightsail_instance.go @@ -4,7 +4,6 @@ import ( "fmt" "strings" - "github.com/infracost/infracost/internal/logging" "github.com/infracost/infracost/internal/resources" "github.com/infracost/infracost/internal/schema" @@ -30,37 +29,35 @@ func (r *LightsailInstance) PopulateUsage(u *schema.UsageData) { } func (r *LightsailInstance) BuildResource() *schema.Resource { - type bundleSpecs struct { - vcpu string - memory string + bundlePrefixMappings := map[string]string{ + "nano": "0.5GB", + "micro": "1GB", + "small": "2GB", + "medium": "4GB", + "large": "8GB", + "xlarge": "16GB", + "2xlarge": "32GB", + "4xlarge": "64GB", } - bundlePrefixMappings := map[string]bundleSpecs{ - "nano": {"2", "0.5GB"}, - "micro": {"2", "1GB"}, - "small": {"2", "2GB"}, - "medium": {"2", "4GB"}, - "large": {"2", "8GB"}, - "xlarge": {"4", "16GB"}, - "2xlarge": {"8", "32GB"}, - } - - operatingSystem := "Linux" + operatingSystemSuffix := "" operatingSystemLabel := "Linux/UNIX" if strings.Contains(strings.ToLower(r.BundleID), "_win_") { - operatingSystem = "Windows" + operatingSystemSuffix = "_win" operatingSystemLabel = "Windows" } bundlePrefix := strings.Split(strings.ToLower(r.BundleID), "_")[0] - specs, ok := bundlePrefixMappings[bundlePrefix] + memory, ok := bundlePrefixMappings[bundlePrefix] if !ok { - logging.Logger.Warn().Msgf("Skipping resource %s. Unrecognized bundle_id %s", r.Address, r.BundleID) - return nil + // this will end up showing a 'product not found' warning + memory = bundlePrefix } + usagetypeRegex := fmt.Sprintf("-BundleUsage:%s%s$", memory, operatingSystemSuffix) + return &schema.Resource{ Name: r.Address, CostComponents: []*schema.CostComponent{ @@ -75,14 +72,9 @@ func (r *LightsailInstance) BuildResource() *schema.Resource { Service: strPtr("AmazonLightsail"), ProductFamily: strPtr("Lightsail Instance"), AttributeFilters: []*schema.AttributeFilter{ - {Key: "operatingSystem", Value: strPtr(operatingSystem)}, - {Key: "vcpu", Value: strPtr(specs.vcpu)}, - {Key: "memory", Value: strPtr(specs.memory)}, + {Key: "usagetype", ValueRegex: regexPtr(usagetypeRegex)}, }, }, - PriceFilter: &schema.PriceFilter{ - EndUsageAmount: strPtr("Inf"), - }, }, }, UsageSchema: r.UsageSchema(), diff --git a/internal/resources/aws/s3_intelligent_tiering_storage_class.go b/internal/resources/aws/s3_intelligent_tiering_storage_class.go index 56161804cb0..f0b955a6b39 100644 --- a/internal/resources/aws/s3_intelligent_tiering_storage_class.go +++ b/internal/resources/aws/s3_intelligent_tiering_storage_class.go @@ -60,8 +60,8 @@ func (a *S3IntelligentTieringStorageClass) BuildResource() *schema.Resource { CostComponents: []*schema.CostComponent{ s3StorageCostComponent("Storage (frequent access)", "AmazonS3", a.Region, "TimedStorage-INT-FA-ByteHrs", a.FrequentAccessStorageGB), s3StorageCostComponent("Storage (infrequent access)", "AmazonS3", a.Region, "TimedStorage-INT-IA-ByteHrs", a.InfrequentAccessStorageGB), - s3StorageVolumeTypeCostComponent("Storage (archive access)", "AmazonS3", a.Region, "TimedStorage-INT-AA-ByteHrs", "IntelligentTieringArchiveAccess", a.FrequentAccessStorageGB), - s3StorageVolumeTypeCostComponent("Storage (deep archive access)", "AmazonS3", a.Region, "TimedStorage-INT-DAA-ByteHrs", "IntelligentTieringDeepArchiveAccess", a.InfrequentAccessStorageGB), + s3StorageVolumeTypeCostComponent("Storage (archive access)", "AmazonS3", a.Region, "TimedStorage-INT-AA-ByteHrs", "IntelligentTieringArchive", a.FrequentAccessStorageGB), + s3StorageVolumeTypeCostComponent("Storage (deep archive access)", "AmazonS3", a.Region, "TimedStorage-INT-DAA-ByteHrs", "IntelligentTieringDeepArchive", a.InfrequentAccessStorageGB), s3MonitoringCostComponent(a.Region, a.MonitoredObjects), s3ApiCostComponent("PUT, COPY, POST, LIST requests", "AmazonS3", a.Region, "Requests-INT-Tier1", a.MonthlyTier1Requests), s3ApiCostComponent("GET, SELECT, and all other requests", "AmazonS3", a.Region, "Requests-INT-Tier2", a.MonthlyTier2Requests), diff --git a/internal/resources/azure/cognitive_account_speech.go b/internal/resources/azure/cognitive_account_speech.go index fff96703ecd..1d0fb7df8f4 100644 --- a/internal/resources/azure/cognitive_account_speech.go +++ b/internal/resources/azure/cognitive_account_speech.go @@ -188,7 +188,7 @@ func (r *CognitiveAccountSpeech) costComponents() []*schema.CostComponent { costComponents = append(costComponents, r.commitmentTierHourlyCostComponents("Speech to text", connectedContainerCommitmentTier, "Commitment Tier Speech to Text Connected", *r.MonthlyConnectedContainerCommitmentSpeechToTextHrs, floatPtrToDecimalPtr(r.MonthlyConnectedContainerCommitmentSpeechToTextOverageHrs))...) } if (r.MonthlyCommitmentSpeechToTextHrs == nil && r.MonthlyConnectedContainerCommitmentSpeechToTextHrs == nil) || r.MonthlySpeechToTextStandardHrs != nil { - costComponents = append(costComponents, r.s0HourlyCostComponent("Speech to text", "Speech To Text", r.MonthlySpeechToTextStandardHrs)) + costComponents = append(costComponents, r.s0HourlyCostComponent("Speech to text", "S1 Speech To Text", r.MonthlySpeechToTextStandardHrs)) } costComponents = append(costComponents, @@ -202,7 +202,7 @@ func (r *CognitiveAccountSpeech) costComponents() []*schema.CostComponent { costComponents = append(costComponents, r.commitmentTierHourlyCostComponents("Speech to text custom model", connectedContainerCommitmentTier, "Commitment Tier Custom Speech to Text Connected", *r.MonthlyConnectedContainerCommitmentSpeechToTextCustomModelHrs, floatPtrToDecimalPtr(r.MonthlyConnectedContainerCommitmentSpeechToTextCustomModelOverageHrs))...) } if (r.MonthlyCommitmentSpeechToTextCustomModelHrs == nil && r.MonthlyConnectedContainerCommitmentSpeechToTextCustomModelHrs == nil) || r.MonthlySpeechToTextCustomModelHrs != nil { - costComponents = append(costComponents, r.s0HourlyCostComponent("Speech to text custom model", "Custom Speech To Text", r.MonthlySpeechToTextCustomModelHrs)) + costComponents = append(costComponents, r.s0HourlyCostComponent("Speech to text custom model", "S1 Custom Speech To Text", r.MonthlySpeechToTextCustomModelHrs)) } costComponents = append(costComponents, []*schema.CostComponent{ diff --git a/internal/resources/google/cloudfunctions_function.go b/internal/resources/google/cloudfunctions_function.go index 5ce60e422a3..98a5fa3ea8d 100644 --- a/internal/resources/google/cloudfunctions_function.go +++ b/internal/resources/google/cloudfunctions_function.go @@ -78,10 +78,10 @@ func (r *CloudFunctionsFunction) BuildResource() *schema.Resource { ProductFilter: &schema.ProductFilter{ VendorName: strPtr("gcp"), Region: strPtr(r.Region), - Service: strPtr("Cloud Functions"), + Service: strPtr("Cloud Run Functions"), ProductFamily: strPtr("ApplicationServices"), AttributeFilters: []*schema.AttributeFilter{ - {Key: "description", Value: strPtr("CPU Time")}, + {Key: "description", ValueRegex: regexPtr("\\(1st Gen\\) CPU Time")}, }, }, UsageBased: true, @@ -94,10 +94,10 @@ func (r *CloudFunctionsFunction) BuildResource() *schema.Resource { ProductFilter: &schema.ProductFilter{ VendorName: strPtr("gcp"), Region: strPtr(r.Region), - Service: strPtr("Cloud Functions"), + Service: strPtr("Cloud Run Functions"), ProductFamily: strPtr("ApplicationServices"), AttributeFilters: []*schema.AttributeFilter{ - {Key: "description", Value: strPtr("Memory Time")}, + {Key: "description", ValueRegex: regexPtr("\\(1st Gen\\) Memory Time")}, }, }, UsageBased: true, @@ -110,10 +110,10 @@ func (r *CloudFunctionsFunction) BuildResource() *schema.Resource { ProductFilter: &schema.ProductFilter{ VendorName: strPtr("gcp"), Region: strPtr("global"), - Service: strPtr("Cloud Functions"), + Service: strPtr("Cloud Run Functions"), ProductFamily: strPtr("ApplicationServices"), AttributeFilters: []*schema.AttributeFilter{ - {Key: "description", Value: strPtr("Invocations")}, + {Key: "description", ValueRegex: regexPtr("\\(1st Gen\\) Invocations")}, }, }, PriceFilter: &schema.PriceFilter{ @@ -129,10 +129,10 @@ func (r *CloudFunctionsFunction) BuildResource() *schema.Resource { ProductFilter: &schema.ProductFilter{ VendorName: strPtr("gcp"), Region: strPtr(r.Region), - Service: strPtr("Cloud Functions"), + Service: strPtr("Cloud Run Functions"), ProductFamily: strPtr("ApplicationServices"), AttributeFilters: []*schema.AttributeFilter{ - {Key: "description", ValueRegex: regexPtr("^Network Data Transfer Out")}, + {Key: "description", ValueRegex: regexPtr("\\(1st Gen\\) Network Data Transfer Out")}, }, }, PriceFilter: &schema.PriceFilter{ diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index 7fcaf3ff292..b79f18cf777 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -219,6 +219,15 @@ func ConfigureTestToCaptureLogs(t *testing.T, runCtx *config.RunContext, level s return logBuf } +func ConfigureTestToFailOnLogs(t *testing.T, runCtx *config.RunContext) { + runCtx.Config.LogLevel = "warn" + runCtx.Config.SetLogDisableTimestamps(true) + runCtx.Config.SetLogWriter(io.MultiWriter(os.Stderr, ErrorOnAnyWriter{t})) + + err := logging.ConfigureBaseLogger(runCtx.Config) + require.Nil(t, err) +} + // From https://gist.github.com/stoewer/fbe273b711e6a06315d19552dd4d33e6 func toSnakeCase(s string) string { var res = make([]rune, 0, len(s)) From eb8a603f2c7ca4de882ebf41aede5baef4b09c6e Mon Sep 17 00:00:00 2001 From: Owen Date: Wed, 4 Dec 2024 17:29:31 +0000 Subject: [PATCH 52/81] fix: Don't leak URL based credentials (#3265) * fix: Don't leak URL based credentials When reporting errors for failures to download sources we need to ensure that the diagnostic error doesn't contain the credentials in the URL - add a redaction for the source - add tests to validate the redaction * chore: clean up the url in the error message too * chore: run format * fix: support nil string in RedactUrlPtr --- internal/hcl/modules/fetch.go | 15 +++--- internal/hcl/modules/registry.go | 9 ++-- internal/schema/project.go | 11 +++- internal/util/redaction.go | 18 +++++++ internal/util/redaction_test.go | 92 ++++++++++++++++++++++++++++++++ 5 files changed, 132 insertions(+), 13 deletions(-) create mode 100644 internal/util/redaction.go create mode 100644 internal/util/redaction_test.go diff --git a/internal/hcl/modules/fetch.go b/internal/hcl/modules/fetch.go index 63610890929..21863c79c55 100644 --- a/internal/hcl/modules/fetch.go +++ b/internal/hcl/modules/fetch.go @@ -3,6 +3,7 @@ package modules import ( "context" "fmt" + "github.com/infracost/infracost/internal/util" "net/url" "os" "path/filepath" @@ -85,12 +86,12 @@ func (p *PackageFetcher) Fetch(moduleAddr string, dest string) error { fetched, err := p.fetchFromLocalCache(moduleAddr, dest) if fetched { - p.logger.Trace().Msgf("cache hit (local): %s", moduleAddr) + p.logger.Trace().Msgf("cache hit (local): %s", util.RedactUrl(moduleAddr)) return nil } if err != nil { - p.logger.Warn().Msgf("error fetching module %s from local cache: %s", moduleAddr, err) + p.logger.Warn().Msgf("error fetching module %s from local cache: %s", util.RedactUrl(moduleAddr), err) } fetched, err = p.fetchFromRemoteCache(moduleAddr, dest) @@ -101,24 +102,24 @@ func (p *PackageFetcher) Fetch(moduleAddr string, dest string) error { } if err != nil { - p.logger.Warn().Msgf("error fetching module %s from remote cache: %s", moduleAddr, err) + p.logger.Warn().Msgf("error fetching module %s from remote cache: %s", util.RedactUrl(moduleAddr), err) } p.logger.Trace().Msgf("cache miss: %s", moduleAddr) _, err = p.fetchFromRemote(moduleAddr, dest) if err != nil { - return fmt.Errorf("error fetching module %s from remote: %w", moduleAddr, err) + return fmt.Errorf("error fetching module %s from remote: %w", util.RedactUrl(moduleAddr), err) } p.localCache.Store(moduleAddr, dest) if p.remoteCache != nil { ttl := determineTTL(moduleAddr) - p.logger.Debug().Msgf("putting module %s into remote cache with ttl %s", moduleAddr, ttl) + p.logger.Debug().Msgf("putting module %s into remote cache with ttl %s", util.RedactUrl(moduleAddr), ttl) err = p.remoteCache.Put(moduleAddr, dest, ttl) if err != nil { - p.logger.Warn().Msgf("error putting module %s into remote cache: %s", moduleAddr, err) + p.logger.Warn().Msgf("error putting module %s into remote cache: %s", util.RedactUrl(moduleAddr), err) } } @@ -137,7 +138,7 @@ func (p *PackageFetcher) fetchFromLocalCache(moduleAddr, dest string) (bool, err return true, nil } - p.logger.Debug().Msgf("module %s already downloaded, copying from '%s' to '%s'", moduleAddr, prevDest, dest) + p.logger.Debug().Msgf("module %s already downloaded, copying from '%s' to '%s'", util.RedactUrl(moduleAddr), prevDest, dest) err := os.Mkdir(dest, os.ModePerm) if err != nil { diff --git a/internal/hcl/modules/registry.go b/internal/hcl/modules/registry.go index 645fe549602..7c1c97eff51 100644 --- a/internal/hcl/modules/registry.go +++ b/internal/hcl/modules/registry.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/infracost/infracost/internal/util" "io" "net/http" "net/url" @@ -138,7 +139,7 @@ func (d *Disco) DownloadLocation(moduleURL RegistryURL, version string) (string, resp, err := d.httpClient.Do(retryReq) if err != nil { - return "", fmt.Errorf("error fetching download URL '%s': %w", downloadURL.String(), err) + return "", fmt.Errorf("error fetching download URL '%s': %w", util.RedactUrl(downloadURL.String()), err) } defer resp.Body.Close() @@ -190,7 +191,7 @@ func NewRegistryLoader(packageFetcher *PackageFetcher, disco *Disco, logger zero func (r *RegistryLoader) lookupModule(moduleAddr string, versionConstraints string) (*RegistryLookupResult, error) { registrySource, err := normalizeRegistrySource(moduleAddr) if err != nil { - r.logger.Debug().Err(err).Msgf("module '%s' not detected as registry module", moduleAddr) + r.logger.Debug().Err(err).Msgf("module '%s' not detected as registry module", util.RedactUrl(moduleAddr)) return &RegistryLookupResult{ OK: false, }, nil @@ -199,14 +200,14 @@ func (r *RegistryLoader) lookupModule(moduleAddr string, versionConstraints stri moduleURL, ok, err := r.disco.ModuleLocation(registrySource) if !ok { if err != nil { - r.logger.Debug().Err(err).Msgf("module '%s' not detected as registry module", moduleAddr) + r.logger.Debug().Err(err).Msgf("module '%s' not detected as registry module", util.RedactUrl(moduleAddr)) } return &RegistryLookupResult{ OK: false, }, nil } if err != nil { - return nil, fmt.Errorf("failed to load remote module from given source %s and version constraints %s %w", moduleAddr, versionConstraints, err) + return nil, fmt.Errorf("failed to load remote module from given source %s and version constraints %s %w", util.RedactUrl(moduleAddr), versionConstraints, err) } versions, err := r.fetchModuleVersions(moduleURL) diff --git a/internal/schema/project.go b/internal/schema/project.go index 3ad2d3172fc..b0470ced3d3 100644 --- a/internal/schema/project.go +++ b/internal/schema/project.go @@ -14,6 +14,7 @@ import ( "github.com/go-git/go-git/v5" "github.com/infracost/infracost/internal/logging" + "github.com/infracost/infracost/internal/util" "github.com/infracost/infracost/internal/vcs" ) @@ -106,9 +107,13 @@ func NewDiagJSONParsingFailure(err error) *ProjectDiag { // download failure. This contains additional information about the module source // and the discovered location returned by the registry. func NewPrivateRegistryDiag(source string, moduleLocation *string, err error) *ProjectDiag { + source = util.RedactUrl(source) + moduleLocation = util.RedactUrlPtr(moduleLocation) + errorString := util.RedactUrl(err.Error()) + return newDiag( diagPrivateRegistryModuleDownloadFailure, - fmt.Sprintf("Failed to lookup module %q - %s", source, err), + fmt.Sprintf("Failed to lookup module %q - %s", source, errorString), true, map[string]interface{}{ "moduleLocation": moduleLocation, @@ -126,9 +131,11 @@ func NewFailedDownloadDiagnostic(source string, err error) *ProjectDiag { sourceType = "ssh" } + source = util.RedactUrl(source) + errorString := util.RedactUrl(err.Error()) return newDiag( diagPrivateModuleDownloadFailure, - fmt.Sprintf("Failed to download module %q - %s", source, err), + fmt.Sprintf("Failed to download module %q - %s", source, errorString), true, map[string]interface{}{ "source": sourceType, diff --git a/internal/util/redaction.go b/internal/util/redaction.go new file mode 100644 index 00000000000..0d269898080 --- /dev/null +++ b/internal/util/redaction.go @@ -0,0 +1,18 @@ +package util + +import "regexp" + +var redactionRegex = regexp.MustCompile(`(?im)://(.+):(.+)@`) + +// redact source removes username:password credentials from sources +func RedactUrl(source string) string { + return redactionRegex.ReplaceAllString(source, "://****:****@") +} + +func RedactUrlPtr(moduleLocation *string) *string { + if moduleLocation == nil { + return nil + } + redacted := RedactUrl(*moduleLocation) + return &redacted +} diff --git a/internal/util/redaction_test.go b/internal/util/redaction_test.go new file mode 100644 index 00000000000..052c60c8f34 --- /dev/null +++ b/internal/util/redaction_test.go @@ -0,0 +1,92 @@ +package util + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRedactUrl(t *testing.T) { + tests := []struct { + name string + url string + expected string + }{ + { + name: "github with secrets", + url: "https://user:password@github.com/some/repo", + expected: "https://****:****@github.com/some/repo", + }, + { + name: "github without secrets", + url: "https://github.com/some/repo", + expected: "https://github.com/some/repo", + }, + { + name: "gitlab ssh with secrets", + url: "ssh://user:password@gitlab.com:22", + expected: "ssh://****:****@gitlab.com:22", + }, + { + name: "gitlab ssh without secrets", + url: "ssh://gitlab.com:22", + expected: "ssh://gitlab.com:22", + }, + { + name: "git error string", + url: "Error downloading ssh://user:password@gitlab.com:22 because it couldn't be found", + expected: "Error downloading ssh://****:****@gitlab.com:22 because it couldn't be found", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, RedactUrl(tt.url)) + }) + } +} + +func TestRedactUrlPtr(t *testing.T) { + tests := []struct { + name string + url *string + expected *string + }{ + { + name: "nil", + url: nil, + expected: nil, + }, + { + name: "github with secrets", + url: stringPtr("https://user:password@github.com/some/repo"), + expected: stringPtr("https://****:****@github.com/some/repo"), + }, + { + name: "github without secrets", + url: stringPtr("https://github.com/some/repo"), + expected: stringPtr("https://github.com/some/repo"), + }, + { + name: "gitlab ssh with secrets", + url: stringPtr("ssh://user:password@gitlab.com:22"), + expected: stringPtr("ssh://****:****@gitlab.com:22"), + }, + { + name: "gitlab ssh without secrets", + url: stringPtr("ssh://gitlab.com:22"), + expected: stringPtr("ssh://gitlab.com:22"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, RedactUrlPtr(tt.url)) + }) + } + +} + +func stringPtr(in string) *string { + return &in +} From 3ad770d987d4e1dd4de1953e3c5d7a0b73122310 Mon Sep 17 00:00:00 2001 From: Hugo Rut Date: Thu, 5 Dec 2024 10:48:27 +0100 Subject: [PATCH 53/81] enhance: improve Terragrunt inputs parsing (#3264) * enhance: improve Terragrunt inputs parsing Improves the Terragrunt parser's ability to provide a partial value for the inputs. Prior to this change any nasty hcl errors within the inputs attribute for a Terragrunt hcl file would mean that we passed empty values to the underlying Terraform module call variables. This change hooks into a new DiagnosticFunc hook (see https://github.com/infracost/terragrunt/pull/19) that allows us to fallback to our parsing/mocking logic for the inputs value. * fix: failing tests because of file func changes * fix: tests and update go mod sha --- cmd/infracost/breakdown_test.go | 15 ++++++ .../breakdown_terraform_file_funcs.golden | 2 +- .../breakdown_terragrunt_file_funcs.golden | 2 +- ...breakdown_terragrunt_partial_inputs.golden | 38 +++++++++++++++ .../modules/example/main.tf | 48 +++++++++++++++++++ .../prod/terragrunt.hcl | 27 +++++++++++ .../terragrunt.hcl | 18 +++++++ go.mod | 4 +- go.sum | 4 +- internal/hcl/attribute.go | 19 ++++++-- internal/hcl/attribute_test.go | 7 +-- internal/hcl/block.go | 6 +-- internal/hcl/funcs/filesystem.go | 14 +++++- internal/hcl/funcs/filesystem_test.go | 12 ++--- .../terraform/terragrunt_hcl_provider.go | 35 +++++++++++++- 15 files changed, 226 insertions(+), 25 deletions(-) create mode 100644 cmd/infracost/testdata/breakdown_terragrunt_partial_inputs/breakdown_terragrunt_partial_inputs.golden create mode 100644 cmd/infracost/testdata/breakdown_terragrunt_partial_inputs/modules/example/main.tf create mode 100644 cmd/infracost/testdata/breakdown_terragrunt_partial_inputs/prod/terragrunt.hcl create mode 100644 cmd/infracost/testdata/breakdown_terragrunt_partial_inputs/terragrunt.hcl diff --git a/cmd/infracost/breakdown_test.go b/cmd/infracost/breakdown_test.go index 0c69dc74e73..f2648a647c7 100644 --- a/cmd/infracost/breakdown_test.go +++ b/cmd/infracost/breakdown_test.go @@ -1565,6 +1565,21 @@ func TestBreakdownTerragruntAutodetectionConfigFileOutput(t *testing.T) { ) } +func TestBreakdownTerragruntPartialInputs(t *testing.T) { + testName := testutil.CalcGoldenFileTestdataDirName() + dir := path.Join("./testdata", testName) + GoldenFileCommandTest( + t, + testutil.CalcGoldenFileTestdataDirName(), + []string{ + "breakdown", + "--path", dir, + "--log-level", "info", + }, + &GoldenFileOptions{LogLevel: strPtr("info"), IgnoreNonGraph: true}, + ) +} + func strPtr(s string) *string { return &s } diff --git a/cmd/infracost/testdata/breakdown_terraform_file_funcs/breakdown_terraform_file_funcs.golden b/cmd/infracost/testdata/breakdown_terraform_file_funcs/breakdown_terraform_file_funcs.golden index e30eb560a4d..dc86aa1aae5 100644 --- a/cmd/infracost/testdata/breakdown_terraform_file_funcs/breakdown_terraform_file_funcs.golden +++ b/cmd/infracost/testdata/breakdown_terraform_file_funcs/breakdown_terraform_file_funcs.golden @@ -128,7 +128,7 @@ Project: main └─ Storage (general purpose SSD, gp2) 8 GB $0.80 module.mod_files.aws_instance.file["pd"] - ├─ Instance usage (Linux/UNIX, on-demand, ) 730 hours not found + ├─ Instance usage (Linux/UNIX, on-demand, instance_type-infracost-mock-44956be29f34) 730 hours not found └─ root_block_device └─ Storage (general purpose SSD, gp2) 8 GB $0.80 diff --git a/cmd/infracost/testdata/breakdown_terragrunt_file_funcs/breakdown_terragrunt_file_funcs.golden b/cmd/infracost/testdata/breakdown_terragrunt_file_funcs/breakdown_terragrunt_file_funcs.golden index e30eb560a4d..dc86aa1aae5 100644 --- a/cmd/infracost/testdata/breakdown_terragrunt_file_funcs/breakdown_terragrunt_file_funcs.golden +++ b/cmd/infracost/testdata/breakdown_terragrunt_file_funcs/breakdown_terragrunt_file_funcs.golden @@ -128,7 +128,7 @@ Project: main └─ Storage (general purpose SSD, gp2) 8 GB $0.80 module.mod_files.aws_instance.file["pd"] - ├─ Instance usage (Linux/UNIX, on-demand, ) 730 hours not found + ├─ Instance usage (Linux/UNIX, on-demand, instance_type-infracost-mock-44956be29f34) 730 hours not found └─ root_block_device └─ Storage (general purpose SSD, gp2) 8 GB $0.80 diff --git a/cmd/infracost/testdata/breakdown_terragrunt_partial_inputs/breakdown_terragrunt_partial_inputs.golden b/cmd/infracost/testdata/breakdown_terragrunt_partial_inputs/breakdown_terragrunt_partial_inputs.golden new file mode 100644 index 00000000000..47285cdbc66 --- /dev/null +++ b/cmd/infracost/testdata/breakdown_terragrunt_partial_inputs/breakdown_terragrunt_partial_inputs.golden @@ -0,0 +1,38 @@ +Project: prod +Module path: prod + + Name Monthly Qty Unit Monthly Cost + + aws_instance.web_app + ├─ Instance usage (Linux/UNIX, on-demand, m5.4xlarge) 730 hours $560.64 + ├─ root_block_device + │ └─ Storage (general purpose SSD, gp2) 100 GB $10.00 + └─ ebs_block_device[0] + ├─ Storage (provisioned IOPS SSD, io1) 1,000 GB $125.00 + └─ Provisioned IOPS 800 IOPS $52.00 + + aws_lambda_function.hello_world + ├─ Requests Monthly cost depends on usage: $0.20 per 1M requests + ├─ Ephemeral storage Monthly cost depends on usage: $0.0000000309 per GB-seconds + └─ Duration (first 6B) Monthly cost depends on usage: $0.0000166667 per GB-seconds + + OVERALL TOTAL $747.64 + +*Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. + +────────────────────────────────── +2 cloud resources were detected: +∙ 2 were estimated + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ +┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ +┃ prod ┃ $748 ┃ - ┃ $748 ┃ +┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ + +Err: + + +Logs: +INFO Autodetected 1 Terragrunt project across 1 root module +INFO Found Terragrunt project prod at directory prod diff --git a/cmd/infracost/testdata/breakdown_terragrunt_partial_inputs/modules/example/main.tf b/cmd/infracost/testdata/breakdown_terragrunt_partial_inputs/modules/example/main.tf new file mode 100644 index 00000000000..4c3840e4c00 --- /dev/null +++ b/cmd/infracost/testdata/breakdown_terragrunt_partial_inputs/modules/example/main.tf @@ -0,0 +1,48 @@ +variable "instance_type" { + description = "The EC2 instance type for the web app" + type = string +} + +variable "root_block_device_volume_size" { + description = "The size of the root block device volume for the web app EC2 instance" + type = number +} + +variable "block_device_volume_size" { + description = "The size of the block device volume for the web app EC2 instance" + type = number +} + +variable "block_device_iops" { + description = "The number of IOPS for the block device for the web app EC2 instance" + type = number +} + +variable "hello_world_function_memory_size" { + description = "The memory to allocate to the hello world Lambda function" + type = number +} + +resource "aws_instance" "web_app" { + ami = "ami-674cbc1e" + instance_type = var.instance_type + + root_block_device { + volume_size = var.root_block_device_volume_size + } + + ebs_block_device { + device_name = "my_data" + volume_type = "io1" + volume_size = var.block_device_volume_size + iops = var.block_device_iops + } +} + +resource "aws_lambda_function" "hello_world" { + function_name = "hello_world" + role = "arn:aws:lambda:us-east-1:aws:resource-id" + handler = "exports.test" + runtime = "nodejs12.x" + memory_size = var.hello_world_function_memory_size +} diff --git a/cmd/infracost/testdata/breakdown_terragrunt_partial_inputs/prod/terragrunt.hcl b/cmd/infracost/testdata/breakdown_terragrunt_partial_inputs/prod/terragrunt.hcl new file mode 100644 index 00000000000..4537ee30987 --- /dev/null +++ b/cmd/infracost/testdata/breakdown_terragrunt_partial_inputs/prod/terragrunt.hcl @@ -0,0 +1,27 @@ +include { + path = find_in_parent_folders() +} + +terraform { + source = "..//modules/example" +} + +locals { + test = yamldecode(sops_decrypt_file("test.yml")) + test2 = jsondecode(sops_decrypt_file("test.json")) + test3 = run_cmd("echo", "hello") +} + +inputs = merge( + yamldecode(file("doesnotexist")).foo, + yamldecode(file("badfile")).bar, + { + some_bad_input: local.test3.output + instance_type = "m5.4xlarge" + root_block_device_volume_size = 100 + block_device_volume_size = 1000 + block_device_iops = 800 + + hello_world_function_memory_size = 1024 + } +) diff --git a/cmd/infracost/testdata/breakdown_terragrunt_partial_inputs/terragrunt.hcl b/cmd/infracost/testdata/breakdown_terragrunt_partial_inputs/terragrunt.hcl new file mode 100644 index 00000000000..b662c9f9acd --- /dev/null +++ b/cmd/infracost/testdata/breakdown_terragrunt_partial_inputs/terragrunt.hcl @@ -0,0 +1,18 @@ +locals { + aws_region = "us-east-1" # <<<<< Try changing this to eu-west-1 to compare the costs +} + +# Generate an AWS provider block +generate "provider" { + path = "provider.tf" + if_exists = "overwrite_terragrunt" + contents = < github.com/aliscott/go-pretty/v6 v6.1 replace github.com/spf13/cobra => github.com/spf13/cobra v1.4.0 -replace github.com/gruntwork-io/terragrunt => github.com/infracost/terragrunt v0.47.1-0.20241113223044-d35392684463 +replace github.com/gruntwork-io/terragrunt => github.com/infracost/terragrunt v0.47.1-0.20241204180911-19aacb9f0819 replace github.com/heimdalr/dag => github.com/aliscott/dag v1.3.2-0.20231115114512-4ce18c825f94 diff --git a/go.sum b/go.sum index 7f1b9748dcc..655be61008e 100644 --- a/go.sum +++ b/go.sum @@ -820,8 +820,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/infracost/go-getter v0.0.0-20240909111353-c0d2eebadfd5 h1:FiK2b8h6CezRGI6CGs7YDVG9nbF2TQGMcRK9iSM37so= github.com/infracost/go-getter v0.0.0-20240909111353-c0d2eebadfd5/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= -github.com/infracost/terragrunt v0.47.1-0.20241113223044-d35392684463 h1:vE7CFFOLqsCgYjeUJElbt5MoWEkWIZxxXGK2Wu1klu4= -github.com/infracost/terragrunt v0.47.1-0.20241113223044-d35392684463/go.mod h1:504J5iD5AjGgP5IwHkUWAcfoqvZIhrR1aEvyxDxX1VA= +github.com/infracost/terragrunt v0.47.1-0.20241204180911-19aacb9f0819 h1:vh0SyaOOktlG7BPv0aG3VBVOexEyY+H/yONfCX5Tum8= +github.com/infracost/terragrunt v0.47.1-0.20241204180911-19aacb9f0819/go.mod h1:504J5iD5AjGgP5IwHkUWAcfoqvZIhrR1aEvyxDxX1VA= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= diff --git a/internal/hcl/attribute.go b/internal/hcl/attribute.go index d93da02cf5d..d5d343379d0 100644 --- a/internal/hcl/attribute.go +++ b/internal/hcl/attribute.go @@ -9,11 +9,12 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclsyntax" - "github.com/infracost/infracost/internal/hcl/mock" "github.com/rs/zerolog" "github.com/zclconf/go-cty/cty" "github.com/zclconf/go-cty/cty/convert" "github.com/zclconf/go-cty/cty/gocty" + + "github.com/infracost/infracost/internal/hcl/mock" ) var ( @@ -48,8 +49,8 @@ type Attribute struct { // Verbose defines if the attribute should log verbose diagnostics messages to debug. Verbose bool Logger zerolog.Logger - // isGraph is a flag that indicates if the attribute should be evaluated with the graph evaluation - isGraph bool + // IsGraph is a flag that indicates if the attribute should be evaluated with the graph evaluation + IsGraph bool // newMock generates a mock value for the attribute if it's value is missing. newMock func(attr *Attribute) cty.Value previousValue cty.Value @@ -135,7 +136,7 @@ func (attr *Attribute) Value() cty.Value { attr.Logger.Trace().Msg("fetching attribute value") var val cty.Value - if attr.isGraph { + if attr.IsGraph { val = attr.graphValue() } else { val = attr.value(0) @@ -258,7 +259,7 @@ func (attr *Attribute) HasChanged() (change bool) { previous := attr.previousValue var current cty.Value - if attr.isGraph { + if attr.IsGraph { current = attr.graphValue() } else { current = attr.value(0) @@ -668,6 +669,14 @@ func mockExpressionCalls(expr hcl.Expression, diagnostics hcl.Diagnostics, mocke SrcRange: t.SrcRange, } case *hclsyntax.TemplateExpr: + for _, d := range diagnostics { + if t == d.Expression { + return &hclsyntax.LiteralValueExpr{ + Val: mockedVal, + } + } + } + newParts := make([]hclsyntax.Expression, len(t.Parts)) for i, part := range t.Parts { newParts[i] = mockExpressionCalls(part, diagnostics, mockedVal) diff --git a/internal/hcl/attribute_test.go b/internal/hcl/attribute_test.go index ab8d7a4c6d6..1eb784d764a 100644 --- a/internal/hcl/attribute_test.go +++ b/internal/hcl/attribute_test.go @@ -8,11 +8,12 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclparse" - "github.com/infracost/infracost/internal/hcl/mock" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/zclconf/go-cty/cty" + + "github.com/infracost/infracost/internal/hcl/mock" ) func TestAttribute_AsInt(t *testing.T) { @@ -150,7 +151,7 @@ locals { Variables: map[string]cty.Value{}, }}, Logger: discard, - isGraph: true, + IsGraph: true, } attr := Attribute{ @@ -171,7 +172,7 @@ locals { }, Verbose: false, Logger: logger, - isGraph: true, + IsGraph: true, } v := attr.Value() diff --git a/internal/hcl/block.go b/internal/hcl/block.go index 050482119dd..d71907b98b2 100644 --- a/internal/hcl/block.go +++ b/internal/hcl/block.go @@ -887,7 +887,7 @@ func (b *Block) GetAttributes() []*Attribute { Logger: b.logger.With().Str( "attribute_name", k, ).Logger(), - isGraph: b.isGraph, + IsGraph: b.isGraph, }) continue } @@ -900,7 +900,7 @@ func (b *Block) GetAttributes() []*Attribute { Logger: b.logger.With().Str( "attribute_name", hclAttributes[k].Name, ).Logger(), - isGraph: b.isGraph, + IsGraph: b.isGraph, }) } @@ -958,7 +958,7 @@ func (b *Block) syntheticAttribute(name string, val cty.Value) *Attribute { Logger: b.logger.With().Str( "attribute_name", name, ).Logger(), - isGraph: b.isGraph, + IsGraph: b.isGraph, } } diff --git a/internal/hcl/funcs/filesystem.go b/internal/hcl/funcs/filesystem.go index 30bbcace364..cce6551c27c 100644 --- a/internal/hcl/funcs/filesystem.go +++ b/internal/hcl/funcs/filesystem.go @@ -29,7 +29,19 @@ func MakeFileFunc(baseDir string, encBase64 bool) function.Function { return cty.StringVal(""), err } - return ff.Call(args) + c, err := ff.Call(args) + + // if we get an error calling the underlying file function this is likely because the path + // argument has been transformed at some point because of infracost mocking/state fallbacks + // so we return a blank string instead of failing the evaluation. This is safer than returning + // an error as we in complex expression cases we can actually return a partial value instead + // of a unknown value which will cause subsequent evaluations to fail. + if err != nil { + logging.Logger.Debug().Msgf("error calling file func: %s returning a blank string for filesytem func", err) + return cty.StringVal(""), nil + } + + return c, nil }, }) } diff --git a/internal/hcl/funcs/filesystem_test.go b/internal/hcl/funcs/filesystem_test.go index 921c8528297..f18d46c7a9a 100644 --- a/internal/hcl/funcs/filesystem_test.go +++ b/internal/hcl/funcs/filesystem_test.go @@ -25,14 +25,14 @@ func TestFile(t *testing.T) { { cty.StringVal("testdata/icon.png"), func(t *testing.T) {}, - cty.NilVal, - true, // Not valid UTF-8 + cty.StringVal(""), + false, }, { cty.StringVal("testdata/missing"), func(t *testing.T) {}, - cty.NilVal, - true, // no file exists + cty.StringVal(""), + false, }, { cty.StringVal("testdata/hello.txt"), @@ -472,8 +472,8 @@ func TestFileBase64(t *testing.T) { }, { cty.StringVal("testdata/missing"), - cty.NilVal, - true, // no file exists + cty.StringVal(""), + false, }, } diff --git a/internal/providers/terraform/terragrunt_hcl_provider.go b/internal/providers/terraform/terragrunt_hcl_provider.go index f13976af6fa..84dc0fb42a5 100644 --- a/internal/providers/terraform/terragrunt_hcl_provider.go +++ b/internal/providers/terraform/terragrunt_hcl_provider.go @@ -32,7 +32,6 @@ import ( hcl2 "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/gohcl" "github.com/hashicorp/hcl/v2/hclparse" - "github.com/infracost/infracost/internal/hcl/mock" "github.com/rs/zerolog" "github.com/sirupsen/logrus" "github.com/zclconf/go-cty/cty" @@ -40,6 +39,8 @@ import ( "github.com/zclconf/go-cty/cty/gocty" ctyJson "github.com/zclconf/go-cty/cty/json" + "github.com/infracost/infracost/internal/hcl/mock" + "github.com/infracost/infracost/internal/clierror" "github.com/infracost/infracost/internal/config" "github.com/infracost/infracost/internal/hcl" @@ -490,6 +491,38 @@ func (p *TerragruntHCLProvider) prepWorkingDirs() ([]*terragruntWorkingDirInfo, }, Parallelism: 1, GetOutputs: p.terragruntPathToValue, + DiagnosticsFunc: func(_ error, filename string, config interface{}, evalContext *hcl2.EvalContext) { + configFile := config.(*tgconfig.TerragruntConfigFile) + f, err := hclparse.NewParser().ParseHCLFile(filename) + if err != nil { + p.logger.Warn().Msgf("Terragrunt diagnostic func failed to reparse Terragrunt config file %s: %s", filename, err) + return + } + + content, _, err := f.Body.PartialContent(&hcl2.BodySchema{ + Attributes: []hcl2.AttributeSchema{ + { + Name: "inputs", + }, + }, + }) + if err != nil { + p.logger.Debug().Msgf("Terragrunt diagnostic func failed to reparse Terragrunt inputs for %s: %s", filename, err) + return + } + + attr := hcl.Attribute{ + HCLAttr: content.Attributes["inputs"], + Ctx: hcl.NewContext(evalContext, nil, p.logger), + Logger: p.logger, + // set is graph to true so that we use the better expression mocking + // when the expression hits a diagnostic. + IsGraph: true, + } + v := attr.Value() + + configFile.Inputs = &v + }, } howThesePathsWereFound := fmt.Sprintf("Terragrunt config file found in a subdirectory of %s", terragruntOptions.WorkingDir) From 85af7d48c5b334d6c78ec6988a8e0d44f3b3bc0c Mon Sep 17 00:00:00 2001 From: Tim McFadden <52185+tim775@users.noreply.github.com> Date: Thu, 5 Dec 2024 20:57:34 -0500 Subject: [PATCH 54/81] fix: typo (#3266) --- .../security_center_subscription_pricing_test.golden | 4 ++-- .../resources/azure/security_center_subscription_pricing.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/providers/terraform/azure/testdata/security_center_subscription_pricing_test/security_center_subscription_pricing_test.golden b/internal/providers/terraform/azure/testdata/security_center_subscription_pricing_test/security_center_subscription_pricing_test.golden index 578962269a2..a49f223a987 100644 --- a/internal/providers/terraform/azure/testdata/security_center_subscription_pricing_test/security_center_subscription_pricing_test.golden +++ b/internal/providers/terraform/azure/testdata/security_center_subscription_pricing_test/security_center_subscription_pricing_test.golden @@ -105,5 +105,5 @@ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ Logs: -WARN Skipping resource azurerm_security_center_subscription_pricing.standard_example["CloudPosture"]. Unknown resource tyoe 'CloudPosture' -WARN Skipping resource azurerm_security_center_subscription_pricing.standard_example_with_usage["CloudPosture"]. Unknown resource tyoe 'CloudPosture' \ No newline at end of file +WARN Skipping resource azurerm_security_center_subscription_pricing.standard_example["CloudPosture"]. Unknown resource type 'CloudPosture' +WARN Skipping resource azurerm_security_center_subscription_pricing.standard_example_with_usage["CloudPosture"]. Unknown resource type 'CloudPosture' \ No newline at end of file diff --git a/internal/resources/azure/security_center_subscription_pricing.go b/internal/resources/azure/security_center_subscription_pricing.go index 9d8da90775b..bdacffd95c5 100644 --- a/internal/resources/azure/security_center_subscription_pricing.go +++ b/internal/resources/azure/security_center_subscription_pricing.go @@ -125,7 +125,7 @@ func (r *SecurityCenterSubscriptionPricing) BuildResource() *schema.Resource { case "cosmosdbs": costComponents = []*schema.CostComponent{r.addCosmosDBCostComponent()} default: - logging.Logger.Warn().Msgf("Skipping resource %s. Unknown resource tyoe '%s'", r.Address, r.ResourceType) + logging.Logger.Warn().Msgf("Skipping resource %s. Unknown resource type '%s'", r.Address, r.ResourceType) } return &schema.Resource{ From 29cd89f67e8a507121e3b4d4d85c7b2b15ecf13e Mon Sep 17 00:00:00 2001 From: Liam Galvin Date: Mon, 9 Dec 2024 15:01:34 +0000 Subject: [PATCH 55/81] perf: Performance optimisations + metrics (#3267) * perf: Performance optimisations + metrics + several bugfixes --- .golangci.yml | 1 + cmd/infracost/breakdown.go | 14 ++ cmd/infracost/run.go | 72 +++++-- ...policy_data_upload_plan_json-upload.golden | 2 +- internal/apiclient/policy.go | 5 +- internal/apiclient/pricing.go | 2 +- internal/apiclient/usage.go | 2 +- internal/config/config.go | 3 + internal/hcl/attribute.go | 5 +- internal/hcl/block.go | 2 +- internal/hcl/context.go | 11 +- internal/hcl/evaluator.go | 1 + internal/hcl/graph.go | 179 ++++++++++-------- internal/hcl/graph_vertex_module_call.go | 4 - internal/hcl/graph_vertex_module_exit.go | 2 - internal/hcl/graph_vertex_output.go | 2 - internal/hcl/graph_vertex_provider.go | 2 - internal/hcl/graph_vertex_resource.go | 2 - internal/hcl/graph_vertex_variable.go | 6 +- internal/hcl/modules/fetch.go | 25 ++- internal/hcl/modules/loader.go | 30 ++- internal/hcl/modules/loader_test.go | 3 + internal/hcl/parser.go | 16 +- internal/hcl/project_locator.go | 12 +- internal/metrics/counter.go | 42 ++++ internal/metrics/metrics.go | 113 +++++++++++ internal/metrics/setting.go | 29 +++ internal/metrics/timer.go | 64 +++++++ internal/output/combined.go | 5 + internal/output/table.go | 4 +- internal/prices/prices.go | 13 +- internal/providers/cloudformation/parser.go | 2 +- internal/providers/detect.go | 1 - internal/providers/terraform/hcl_provider.go | 22 +++ internal/providers/terraform/parser.go | 18 +- .../terraform/terragrunt_hcl_provider.go | 5 + .../google/artifact_registry_repository.go | 2 +- internal/schema/project.go | 4 +- internal/usage/usage_file.go | 3 + internal/vcs/metadata.go | 2 +- 40 files changed, 556 insertions(+), 176 deletions(-) create mode 100644 internal/metrics/counter.go create mode 100644 internal/metrics/metrics.go create mode 100644 internal/metrics/setting.go create mode 100644 internal/metrics/timer.go diff --git a/.golangci.yml b/.golangci.yml index c33f285bdb7..77bd0d3cc1e 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -14,6 +14,7 @@ linters: - staticcheck - typecheck - unused + - prealloc linters-settings: gocritic: diff --git a/cmd/infracost/breakdown.go b/cmd/infracost/breakdown.go index 1ca86deff9e..18c8ec91215 100644 --- a/cmd/infracost/breakdown.go +++ b/cmd/infracost/breakdown.go @@ -1,6 +1,9 @@ package main import ( + "fmt" + + "github.com/infracost/infracost/internal/metrics" "github.com/spf13/cobra" "github.com/infracost/infracost/internal/config" @@ -23,6 +26,17 @@ func breakdownCmd(ctx *config.RunContext) *cobra.Command { infracost breakdown --path plan.json`, ValidArgs: []string{"--", "-"}, RunE: checkAPIKeyIsValid(ctx, func(cmd *cobra.Command, args []string) error { + + timer := metrics.GetTimer("breakdown.total_duration", false).Start() + defer func() { + timer.Stop() + if path := ctx.Config.MetricsPath; path != "" { + if err := metrics.WriteMetrics(path); err != nil { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Error writing metrics: %s\n", err) + } + } + }() + if err := checkAPIKey(ctx.Config.APIKey, ctx.Config.PricingAPIEndpoint, ctx.Config.DefaultPricingAPIEndpoint); err != nil { return err } diff --git a/cmd/infracost/run.go b/cmd/infracost/run.go index adb0260db49..67c77099dbb 100644 --- a/cmd/infracost/run.go +++ b/cmd/infracost/run.go @@ -15,6 +15,7 @@ import ( "time" "github.com/Rhymond/go-money" + "github.com/infracost/infracost/internal/metrics" "github.com/spf13/cobra" "golang.org/x/sync/errgroup" @@ -112,10 +113,8 @@ func runMain(cmd *cobra.Command, runCtx *config.RunContext) error { projectContexts := make([]*config.ProjectContext, 0) for _, projectResult := range projectResults { - for _, project := range projectResult.projectOut.projects { - projectContexts = append(projectContexts, projectResult.ctx) - projects = append(projects, project) - } + projectContexts = append(projectContexts, projectResult.ctx) + projects = append(projects, projectResult.projectOut.projects...) } r, err := output.ToOutputFormat(runCtx.Config, projects) @@ -237,6 +236,8 @@ func newParallelRunner(cmd *cobra.Command, runCtx *config.RunContext) (*parallel } runCtx.ContextValues.SetValue("parallelism", parallelism) + metrics.GetCounter("parallel_runner.parallelism", false).Add(parallelism) + return ¶llelRunner{ parallelism: parallelism, runCtx: runCtx, @@ -252,6 +253,9 @@ func (r *parallelRunner) run() ([]projectResult, error) { var totalRootModules int var i int + parallelRunnerTimer := metrics.GetTimer("parallel_runner.run.total_duration", false).Start() + defer parallelRunnerTimer.Stop() + isAuto := r.runCtx.IsAutoDetect() for _, p := range r.runCtx.Config.Projects { detectionOutput, err := providers.Detect(r.runCtx, p, r.prior == nil) @@ -271,6 +275,10 @@ func (r *parallelRunner) run() ([]projectResult, error) { totalRootModules += detectionOutput.RootModules } + + metrics.GetCounter("parallel_runner.project_count", false).Add(i) + metrics.GetCounter("parallel_runner.root_module_count", false).Add(totalRootModules) + projectCounts := make(map[string]int) for _, job := range queue { if job.err != nil { @@ -286,7 +294,7 @@ func (r *parallelRunner) run() ([]projectResult, error) { projectCounts[provider.DisplayType()] = 1 } - var order []string + order := make([]string, 0, len(projectCounts)) for displayType := range projectCounts { order = append(order, displayType) } @@ -363,6 +371,7 @@ func (r *parallelRunner) run() ([]projectResult, error) { errGroup, _ := errgroup.WithContext(context.Background()) for i := 0; i < r.parallelism; i++ { errGroup.Go(func() (err error) { + // defer a function to recover from any panics spawned by child goroutines. // This is done as recover works only in the same goroutine that it is called. // We need to catch any child goroutine panics and hand them up to the main caller @@ -375,30 +384,44 @@ func (r *parallelRunner) run() ([]projectResult, error) { }() for job := range jobs { - var configProjects *projectOutput - ctx := job.ctx - if job.err != nil { - configProjects = newErroredProject(job.provider, job.ctx, job.err) - } else { - configProjects, err = r.runProvider(job) - ctx = job.provider.Context() - if err != nil { - configProjects = newErroredProject(job.provider, ctx, err) + func() { + var metricContext []string + if job.provider != nil { + metricContext = append(metricContext, job.provider.Type()) + if job.provider.Context() != nil { + metricContext = append(metricContext, job.provider.Context().ProjectConfig.Path) + } } + jobTimer := metrics.GetTimer("parallel_runner.job.duration", false, metricContext...).Start() + defer jobTimer.Stop() + + var configProjects *projectOutput + ctx := job.ctx + if job.err != nil { + configProjects = newErroredProject(job.provider, job.ctx, job.err) + } else { + configProjects, err = r.runProvider(job) + ctx = job.provider.Context() + if err != nil { + configProjects = newErroredProject(job.provider, ctx, err) + } - } + } - projectResultChan <- projectResult{ - index: job.index, - ctx: ctx, - projectOut: configProjects, - } + projectResultChan <- projectResult{ + index: job.index, + ctx: ctx, + projectOut: configProjects, + } + }() } return nil }) } + allJobsTimer := metrics.GetTimer("parallel_runner.all_jobs.duration", false).Start() + for _, job := range queue { jobs <- job } @@ -406,6 +429,7 @@ func (r *parallelRunner) run() ([]projectResult, error) { close(jobs) err := errGroup.Wait() + allJobsTimer.Stop() if err != nil { return nil, err } @@ -453,7 +477,9 @@ func (r *parallelRunner) runProvider(job projectJob) (out *projectOutput, err er // Generate usage file if r.runCtx.Config.SyncUsageFile { + usageGenTimer := metrics.GetTimer("parallel_runner.usage_gen.duration", false, path).Start() err = r.generateUsageFile(job.provider) + usageGenTimer.Stop() if err != nil { return nil, fmt.Errorf("Error generating usage file %w", err) } @@ -512,7 +538,9 @@ func (r *parallelRunner) runProvider(job projectJob) (out *projectOutput, err er out = &projectOutput{} t1 := time.Now() + loadResourcesTimer := metrics.GetTimer("parallel_runner.load_resources.duration", false, path).Start() projects, err := job.provider.LoadResources(usageData) + loadResourcesTimer.Stop() if err != nil { r.cmd.PrintErrln() return nil, err @@ -520,8 +548,12 @@ func (r *parallelRunner) runProvider(job projectJob) (out *projectOutput, err er _ = r.uploadCloudResourceIDs(projects) + buildResourcesTimer := metrics.GetTimer("parallel_runner.build_resources.duration", false, path).Start() r.buildResources(projects) + buildResourcesTimer.Stop() + costingTimer := metrics.GetTimer("parallel_runner.costing.duration", false, path).Start() + defer costingTimer.Stop() logging.Logger.Debug().Msg("Retrieving cloud prices to calculate costs") for _, project := range projects { diff --git a/cmd/infracost/testdata/breakdown_with_policy_data_upload_plan_json/breakdown_with_policy_data_upload_plan_json-upload.golden b/cmd/infracost/testdata/breakdown_with_policy_data_upload_plan_json/breakdown_with_policy_data_upload_plan_json-upload.golden index 8f60f6c2923..c86e3eb7fbe 100644 --- a/cmd/infracost/testdata/breakdown_with_policy_data_upload_plan_json/breakdown_with_policy_data_upload_plan_json-upload.golden +++ b/cmd/infracost/testdata/breakdown_with_policy_data_upload_plan_json/breakdown_with_policy_data_upload_plan_json-upload.golden @@ -67,7 +67,7 @@ storePolicyResources: }, "references": [], "infracostMetadata": { - "calls": null, + "calls": [], "checksum": "a1c17a5372d0b56215b34af598a205bb8cdc208b82780e3ba2cac92460b26dd7", "endLine": 0, "filename": "", diff --git a/internal/apiclient/policy.go b/internal/apiclient/policy.go index a196302e1f9..0a629156cff 100644 --- a/internal/apiclient/policy.go +++ b/internal/apiclient/policy.go @@ -250,8 +250,9 @@ func filterResource(rd *schema.ResourceData, al allowList) policy2Resource { return references[i].Key < references[j].Key }) - var mdCalls []policy2InfracostMetadataCall - for _, c := range rd.Metadata["calls"].Array() { + calls := rd.Metadata["calls"].Array() + mdCalls := make([]policy2InfracostMetadataCall, 0, len(calls)) + for _, c := range calls { mdCalls = append(mdCalls, policy2InfracostMetadataCall{ BlockName: c.Get("blockName").String(), EndLine: c.Get("endLine").Int(), diff --git a/internal/apiclient/pricing.go b/internal/apiclient/pricing.go index 7511f366d19..626ea19234c 100644 --- a/internal/apiclient/pricing.go +++ b/internal/apiclient/pricing.go @@ -361,7 +361,7 @@ func (c *PricingAPIClient) PerformRequest(req BatchRequest) ([]PriceQueryResult, } // now we deduplicate the queries, ensuring that a request for a price only happens once. - var deduplicatedServerQueries []pricingQuery + deduplicatedServerQueries := make([]pricingQuery, 0, len(serverQueries)) seenQueries := map[uint64]bool{} for _, query := range serverQueries { if seenQueries[query.hash] { diff --git a/internal/apiclient/usage.go b/internal/apiclient/usage.go index f06d6cc2b51..2399e1dd61a 100644 --- a/internal/apiclient/usage.go +++ b/internal/apiclient/usage.go @@ -177,7 +177,7 @@ func (c *UsageAPIClient) buildActualCostsQuery(vars ActualCostsQueryVariables) G // ListUsageQuantities queries the Infracost Cloud Usage API to retrieve usage estimates // derived from cloud provider reported usage and costs. func (c *UsageAPIClient) ListUsageQuantities(vars []*UsageQuantitiesQueryVariables) ([]*schema.UsageData, error) { - var queries []GraphQLQuery + queries := make([]GraphQLQuery, 0, len(vars)) for _, v := range vars { logging.Logger.Debug().Msgf("Getting usage quantities from %s for %s %s %v", c.endpoint, v.ResourceType, v.Address, v.UsageKeys) queries = append(queries, c.buildUsageQuantitiesQuery(*v)) diff --git a/internal/config/config.go b/internal/config/config.go index 1f018dd62f0..cd77d872c0e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -163,6 +163,9 @@ type Config struct { S3ModuleCacheBucket string `envconfig:"S3_MODULE_CACHE_BUCKET"` S3ModuleCachePrefix string `envconfig:"S3_MODULE_CACHE_PREFIX"` + // metrics dump path + MetricsPath string `envconfig:"METRICS_PATH"` + // Org settings EnableCloudForOrganization bool diff --git a/internal/hcl/attribute.go b/internal/hcl/attribute.go index d5d343379d0..a052dfae301 100644 --- a/internal/hcl/attribute.go +++ b/internal/hcl/attribute.go @@ -1122,9 +1122,10 @@ func (attr *Attribute) AllReferences() []*Reference { // VerticesReferenced traverses all the Expressions used by the attribute to build a // list of all the Blocks referenced by the Attribute. func (attr *Attribute) VerticesReferenced(b *Block) []VertexReference { - var refs []VertexReference + allRefs := attr.AllReferences() + refs := make([]VertexReference, 0, len(allRefs)) - for _, ref := range attr.AllReferences() { + for _, ref := range allRefs { key := ref.String() if shouldSkipRef(b, attr, key) { diff --git a/internal/hcl/block.go b/internal/hcl/block.go index d71907b98b2..bce235df500 100644 --- a/internal/hcl/block.go +++ b/internal/hcl/block.go @@ -829,7 +829,7 @@ func (b *Block) Children() Blocks { return nil } - var children Blocks + children := make(Blocks, 0, len(b.childBlocks)) for _, child := range b.childBlocks { // Skip lifecycle meta argument blocks since it never needs to be evaluated if supportsLifecycle(child) && child.Type() == "lifecycle" { diff --git a/internal/hcl/context.go b/internal/hcl/context.go index eef8b3bdc1f..a5660ad51d1 100644 --- a/internal/hcl/context.go +++ b/internal/hcl/context.go @@ -101,7 +101,7 @@ func mergeVars(src cty.Value, parts []string, value cty.Value) cty.Value { return value } - data := make(map[string]cty.Value) + var data map[string]cty.Value if src.Type().IsObjectType() && !src.IsNull() && src.LengthInt() > 0 { data = src.AsValueMap() tmp, ok := data[parts[0]] // nolint:gosec // ignore "G602: slice index out of range" since we already check len(parts) == 0 @@ -110,6 +110,8 @@ func mergeVars(src cty.Value, parts []string, value cty.Value) cty.Value { } else { src = tmp } + } else { + data = make(map[string]cty.Value) } data[parts[0]] = mergeVars(src, parts[1:], value) // nolint:gosec // ignore "G602: slice index out of range" since we already check len(parts) == 0 @@ -122,12 +124,7 @@ func mergeVars(src cty.Value, parts []string, value cty.Value) cty.Value { // takes precedence over object `a`, unless both values are valid cty objects, in // which case they are recursively merged. func mergeObjects(a cty.Value, b cty.Value) cty.Value { - output := make(map[string]cty.Value) - - for key, val := range a.AsValueMap() { - output[key] = val - } - + output := a.AsValueMap() for key, val := range b.AsValueMap() { old, exists := output[key] diff --git a/internal/hcl/evaluator.go b/internal/hcl/evaluator.go index dd5bc452d74..85369d1f702 100644 --- a/internal/hcl/evaluator.go +++ b/internal/hcl/evaluator.go @@ -239,6 +239,7 @@ func (e *Evaluator) MissingVars() []string { // this Module. func (e *Evaluator) Run() (*Module, error) { var lastContext hcl.EvalContext + // first we need to evaluate the top level Context - so this can be passed to any child modules that are found. e.logger.Debug().Msg("evaluating top level context") e.evaluate(lastContext) diff --git a/internal/hcl/graph.go b/internal/hcl/graph.go index 73a43d834f1..41edb0e7567 100644 --- a/internal/hcl/graph.go +++ b/internal/hcl/graph.go @@ -23,33 +23,24 @@ type ModuleConfig struct { } type ModuleConfigs struct { - configs map[string][]ModuleConfig - mu sync.RWMutex + configs sync.Map } func NewModuleConfigs() *ModuleConfigs { - return &ModuleConfigs{ - configs: make(map[string][]ModuleConfig), - mu: sync.RWMutex{}, - } + return &ModuleConfigs{} } func (m *ModuleConfigs) Add(moduleAddress string, moduleConfig ModuleConfig) { - m.mu.Lock() - defer m.mu.Unlock() - - if _, ok := m.configs[moduleAddress]; !ok { - m.configs[moduleAddress] = []ModuleConfig{} - } - - m.configs[moduleAddress] = append(m.configs[moduleAddress], moduleConfig) + configs, _ := m.configs.LoadOrStore(moduleAddress, []ModuleConfig{}) + m.configs.Store(moduleAddress, append(configs.([]ModuleConfig), moduleConfig)) } func (m *ModuleConfigs) Get(moduleAddress string) []ModuleConfig { - m.mu.RLock() - defer m.mu.RUnlock() - - return m.configs[moduleAddress] + configs, _ := m.configs.Load(moduleAddress) + if configs == nil { + return nil + } + return configs.([]ModuleConfig) } type Graph struct { @@ -218,32 +209,33 @@ func (g *Graph) Populate(evaluator *Evaluator) error { } } - edges := make([]dag.EdgeInput, 0) - for _, vertex := range vertexes { id := vertex.ID() modAddr := vertex.ModuleAddress() if modAddr == "" { g.logger.Debug().Msgf("adding edge: %s, %s", g.rootVertex.ID(), id) - edges = append(edges, dag.EdgeInput{ - SrcID: g.rootVertex.ID(), - DstID: id, - }) + if ok, _ := g.dag.IsEdge(g.rootVertex.ID(), id); !ok { + if err := g.dag.AddEdge(g.rootVertex.ID(), id); err != nil { + return fmt.Errorf("error adding edge %s, %s %w", g.rootVertex.ID(), id, err) + } + } } else { // Add the module call edge g.logger.Debug().Msgf("adding edge: %s, %s", moduleCallID(modAddr), id) - edges = append(edges, dag.EdgeInput{ - SrcID: moduleCallID(modAddr), - DstID: id, - }) + if ok, _ := g.dag.IsEdge(moduleCallID(modAddr), id); !ok { + if err := g.dag.AddEdge(moduleCallID(modAddr), id); err != nil { + return fmt.Errorf("error adding edge %s, %s %w", moduleCallID(modAddr), id, err) + } + } // Add the module exit edge g.logger.Debug().Msgf("adding edge: %s, %s", id, modAddr) - edges = append(edges, dag.EdgeInput{ - SrcID: id, - DstID: modAddr, - }) + if ok, _ := g.dag.IsEdge(id, modAddr); !ok { + if err := g.dag.AddEdge(id, modAddr); err != nil { + return fmt.Errorf("error adding edge %s, %s %w", id, modAddr, err) + } + } } for _, ref := range vertex.References() { @@ -324,11 +316,11 @@ func (g *Graph) Populate(evaluator *Evaluator) error { _, err := g.dag.GetVertex(srcID) if err == nil { g.logger.Debug().Msgf("adding edge: %s, %s", srcID, dstID) - edges = append(edges, dag.EdgeInput{ - SrcID: srcID, - DstID: dstID, - }) - + if ok, _ := g.dag.IsEdge(srcID, dstID); !ok { + if err := g.dag.AddEdge(srcID, dstID); err != nil { + return fmt.Errorf("error adding edge %s, %s %w", srcID, dstID, err) + } + } continue } @@ -342,10 +334,11 @@ func (g *Graph) Populate(evaluator *Evaluator) error { _, err := g.dag.GetVertex(srcID) if err == nil { g.logger.Debug().Msgf("adding edge: %s, %s", srcID, dstID) - edges = append(edges, dag.EdgeInput{ - SrcID: srcID, - DstID: dstID, - }) + if ok, _ := g.dag.IsEdge(srcID, dstID); !ok { + if err := g.dag.AddEdge(srcID, dstID); err != nil { + return fmt.Errorf("error adding edge %s, %s %w", srcID, dstID, err) + } + } continue } @@ -353,11 +346,6 @@ func (g *Graph) Populate(evaluator *Evaluator) error { } } - err = g.dag.AddEdges(edges) - if err != nil { - return fmt.Errorf("error adding edges %w", err) - } - // Setup initial context g.moduleConfigs.Add("", ModuleConfig{ name: "", @@ -385,16 +373,46 @@ func (g *Graph) AsJSON() ([]byte, error) { func (g *Graph) Walk() { v := NewGraphVisitor(g.logger, g.vertexMutex) + TopologicalWalk(g.dag, v.Visit) +} - flowCallback := func(d *dag.DAG, id string, parentResults []dag.FlowResult) (interface{}, error) { - vertex, _ := d.GetVertex(id) - - v.Visit(id, vertex) +func TopologicalWalk(graph *dag.DAG, visitor func(id string, vertex Vertex)) { + type queueItem struct { + id string + vertex Vertex + } - return vertex, nil + // Calculate in-degrees + vertices := graph.GetVertices() + queue := make([]queueItem, 0, len(vertices)) + inDegrees := make(map[string]int, len(vertices)) + for id, vertex := range vertices { + predecessors, _ := graph.GetParents(id) + inDegrees[id] = len(predecessors) + if inDegrees[id] == 0 { + queue = append(queue, queueItem{ + id: id, + vertex: vertex.(Vertex), + }) + } } - _, _ = g.dag.DescendantsFlow(g.rootVertex.ID(), nil, flowCallback) + // Process in topological order + for len(queue) > 0 { + current := queue[0] + queue = queue[1:] + visitor(current.id, current.vertex) + children, _ := graph.GetChildren(current.id) + for id, successor := range children { + inDegrees[id]-- + if inDegrees[id] == 0 { + queue = append(queue, queueItem{ + id: id, + vertex: successor.(Vertex), + }) + } + } + } } func (g *Graph) Run(evaluator *Evaluator) (*Module, error) { @@ -424,11 +442,9 @@ func NewGraphVisitor(logger zerolog.Logger, vertexMutex *sync.Mutex) *GraphVisit } } -func (v *GraphVisitor) Visit(id string, vertex interface{}) { +func (v *GraphVisitor) Visit(id string, vertex Vertex) { v.logger.Debug().Msgf("visiting vertex %q", id) - - vert := vertex.(Vertex) - err := vert.Visit(v.vertexMutex) + err := vertex.Visit(v.vertexMutex) if err != nil { v.logger.Debug().Err(err).Msgf("ignoring vertex %q because an error was encountered", id) } @@ -439,36 +455,33 @@ func (g *Graph) loadAllBlocks(evaluator *Evaluator) ([]*Block, error) { } func (g *Graph) loadBlocksForModule(evaluator *Evaluator) ([]*Block, error) { - var blocks []*Block - - for _, block := range evaluator.module.Blocks { - blocks = append(blocks, block) - - if block.Type() == "module" { - modCall, err := evaluator.loadModule(block) - if err != nil { - return nil, fmt.Errorf("could not load module %q", block.FullName()) - } + blocks := make([]*Block, len(evaluator.module.Blocks)) + copy(blocks, evaluator.module.Blocks) - moduleEvaluator := NewEvaluator( - *modCall.Module, - evaluator.workingDir, - map[string]cty.Value{}, - evaluator.moduleMetadata, - map[string]map[string]cty.Value{}, - evaluator.workspace, - evaluator.blockBuilder, - evaluator.logger, - evaluator.isGraph, - ) - - modBlocks, err := g.loadBlocksForModule(moduleEvaluator) - if err != nil { - return nil, fmt.Errorf("could not load blocks for module %q", block.FullName()) - } + for _, block := range evaluator.module.Blocks.OfType("module") { + modCall, err := evaluator.loadModule(block) + if err != nil { + return nil, fmt.Errorf("could not load module %q", block.FullName()) + } - blocks = append(blocks, modBlocks...) + moduleEvaluator := NewEvaluator( + *modCall.Module, + evaluator.workingDir, + map[string]cty.Value{}, + evaluator.moduleMetadata, + map[string]map[string]cty.Value{}, + evaluator.workspace, + evaluator.blockBuilder, + evaluator.logger, + evaluator.isGraph, + ) + + modBlocks, err := g.loadBlocksForModule(moduleEvaluator) + if err != nil { + return nil, fmt.Errorf("could not load blocks for module %q", block.FullName()) } + + blocks = append(blocks, modBlocks...) } return blocks, nil diff --git a/internal/hcl/graph_vertex_module_call.go b/internal/hcl/graph_vertex_module_call.go index 39fd7b085b2..79abe19325c 100644 --- a/internal/hcl/graph_vertex_module_call.go +++ b/internal/hcl/graph_vertex_module_call.go @@ -78,8 +78,6 @@ func (v *VertexModuleCall) Visit(mutex *sync.Mutex) error { } func (v *VertexModuleCall) evaluate(e *Evaluator, b *Block, mutex *sync.Mutex) error { - mutex.Lock() - defer mutex.Unlock() if b.Label() == "" { return fmt.Errorf("module block %s has no label", b.FullName()) @@ -92,8 +90,6 @@ func (v *VertexModuleCall) evaluate(e *Evaluator, b *Block, mutex *sync.Mutex) e } func (v *VertexModuleCall) expand(e *Evaluator, b *Block, mutex *sync.Mutex) ([]*Block, error) { - mutex.Lock() - defer mutex.Unlock() expanded := []*Block{b} expanded = e.expandBlockForEaches(expanded) expanded = e.expandBlockCounts(expanded) diff --git a/internal/hcl/graph_vertex_module_exit.go b/internal/hcl/graph_vertex_module_exit.go index 5da286c647c..ccca88fdb92 100644 --- a/internal/hcl/graph_vertex_module_exit.go +++ b/internal/hcl/graph_vertex_module_exit.go @@ -25,8 +25,6 @@ func (v *VertexModuleExit) References() []VertexReference { } func (v *VertexModuleExit) Visit(mutex *sync.Mutex) error { - mutex.Lock() - defer mutex.Unlock() moduleInstances := v.moduleConfigs.Get(v.block.FullName()) diff --git a/internal/hcl/graph_vertex_output.go b/internal/hcl/graph_vertex_output.go index 29037244dc4..f8ab9dd463c 100644 --- a/internal/hcl/graph_vertex_output.go +++ b/internal/hcl/graph_vertex_output.go @@ -29,8 +29,6 @@ func (v *VertexOutput) References() []VertexReference { } func (v *VertexOutput) Visit(mutex *sync.Mutex) error { - mutex.Lock() - defer mutex.Unlock() moduleInstances := v.moduleConfigs.Get(v.block.ModuleAddress()) if len(moduleInstances) == 0 { diff --git a/internal/hcl/graph_vertex_provider.go b/internal/hcl/graph_vertex_provider.go index c040568aa1a..10e770e29da 100644 --- a/internal/hcl/graph_vertex_provider.go +++ b/internal/hcl/graph_vertex_provider.go @@ -33,8 +33,6 @@ func (v *VertexProvider) References() []VertexReference { } func (v *VertexProvider) Visit(mutex *sync.Mutex) error { - mutex.Lock() - defer mutex.Unlock() provider := v.block.Label() if provider == "" { diff --git a/internal/hcl/graph_vertex_resource.go b/internal/hcl/graph_vertex_resource.go index d504f38f4a6..9cb5b18428a 100644 --- a/internal/hcl/graph_vertex_resource.go +++ b/internal/hcl/graph_vertex_resource.go @@ -27,8 +27,6 @@ func (v *VertexResource) References() []VertexReference { } func (v *VertexResource) Visit(mutex *sync.Mutex) error { - mutex.Lock() - defer mutex.Unlock() moduleInstances := v.moduleConfigs.Get(v.block.ModuleAddress()) if len(moduleInstances) == 0 { diff --git a/internal/hcl/graph_vertex_variable.go b/internal/hcl/graph_vertex_variable.go index b2dc483e3e8..cd5d41de444 100644 --- a/internal/hcl/graph_vertex_variable.go +++ b/internal/hcl/graph_vertex_variable.go @@ -27,8 +27,6 @@ func (v *VertexVariable) References() []VertexReference { } func (v *VertexVariable) Visit(mutex *sync.Mutex) error { - mutex.Lock() - defer mutex.Unlock() moduleInstances := v.moduleConfigs.Get(v.block.ModuleAddress()) if len(moduleInstances) == 0 { @@ -49,8 +47,8 @@ func (v *VertexVariable) Visit(mutex *sync.Mutex) error { if moduleInstance.moduleCall != nil { attrName := v.block.TypeLabel() - attr, ok := moduleInstance.moduleCall.Definition.AttributesAsMap()[attrName] - if ok { + attr := moduleInstance.moduleCall.Definition.GetAttribute(attrName) + if attr != nil { inputVars = map[string]cty.Value{ attrName: attr.Value(), } diff --git a/internal/hcl/modules/fetch.go b/internal/hcl/modules/fetch.go index 21863c79c55..7d9ee4c32da 100644 --- a/internal/hcl/modules/fetch.go +++ b/internal/hcl/modules/fetch.go @@ -3,7 +3,6 @@ package modules import ( "context" "fmt" - "github.com/infracost/infracost/internal/util" "net/url" "os" "path/filepath" @@ -12,6 +11,8 @@ import ( "sync" "time" + "github.com/infracost/infracost/internal/util" + "github.com/hashicorp/go-getter" "github.com/otiai10/copy" "github.com/rs/zerolog" @@ -29,13 +30,21 @@ const defaultModuleRetrieveTimeout = 3 * time.Minute // PackageFetcher downloads modules from a remote source to the given destination // This supports all the non-local and non-Terraform registry sources listed here: https://www.terraform.io/language/modules/sources type PackageFetcher struct { - localCache sync.Map remoteCache RemoteCache logger zerolog.Logger getters map[string]getter.Getter timeout time.Duration } +// use a global cache to avoid downloading the same module multiple times for each project +var localCache sync.Map +var errorCache sync.Map + +func ResetGlobalModuleCache() { + localCache = sync.Map{} + errorCache = sync.Map{} +} + type PackageFetcherOpts func(*PackageFetcher) // NewPackageFetcher constructs a new package fetcher @@ -97,7 +106,7 @@ func (p *PackageFetcher) Fetch(moduleAddr string, dest string) error { fetched, err = p.fetchFromRemoteCache(moduleAddr, dest) if fetched { p.logger.Trace().Msgf("cache hit (remote): %s", moduleAddr) - p.localCache.Store(moduleAddr, dest) + localCache.Store(moduleAddr, dest) return nil } @@ -112,7 +121,7 @@ func (p *PackageFetcher) Fetch(moduleAddr string, dest string) error { return fmt.Errorf("error fetching module %s from remote: %w", util.RedactUrl(moduleAddr), err) } - p.localCache.Store(moduleAddr, dest) + localCache.Store(moduleAddr, dest) if p.remoteCache != nil { ttl := determineTTL(moduleAddr) @@ -127,7 +136,7 @@ func (p *PackageFetcher) Fetch(moduleAddr string, dest string) error { } func (p *PackageFetcher) fetchFromLocalCache(moduleAddr, dest string) (bool, error) { - v, ok := p.localCache.Load(moduleAddr) + v, ok := localCache.Load(moduleAddr) if !ok { return false, nil } @@ -186,6 +195,11 @@ func (p *PackageFetcher) fetchFromRemoteCache(moduleAddr, dest string) (bool, er } func (p *PackageFetcher) fetchFromRemote(moduleAddr, dest string) (bool, error) { + + if err, ok := errorCache.Load(moduleAddr); ok { + return false, err.(error) + } + decompressors := map[string]getter.Decompressor{} for k, decompressor := range getter.Decompressors { decompressors[k] = decompressor @@ -214,6 +228,7 @@ func (p *PackageFetcher) fetchFromRemote(moduleAddr, dest string) (bool, error) err := client.Get() if err != nil { + errorCache.Store(moduleAddr, err) return false, err } diff --git a/internal/hcl/modules/loader.go b/internal/hcl/modules/loader.go index 5d5076015d0..c726d3db902 100644 --- a/internal/hcl/modules/loader.go +++ b/internal/hcl/modules/loader.go @@ -2,7 +2,7 @@ package modules import ( "bytes" - "crypto/md5" //nolint + "crypto/md5" // nolint "errors" "fmt" "net/url" @@ -16,9 +16,10 @@ import ( "sync" "time" - getter "github.com/hashicorp/go-getter" + "github.com/hashicorp/go-getter" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/terraform-config-inspect/tfconfig" + "github.com/infracost/infracost/internal/metrics" "github.com/otiai10/copy" "github.com/rs/zerolog" giturls "github.com/whilp/git-urls" @@ -122,7 +123,7 @@ func (m *ModuleLoader) manifestFilePath(projectPath string) string { } rel, _ := filepath.Rel(m.cachePath, projectPath) - sum := md5.Sum([]byte(rel)) //nolint + sum := md5.Sum([]byte(rel)) // nolint return filepath.Join(m.cachePath, ".infracost/terraform_modules/", fmt.Sprintf("manifest-%x.json", sum)) } @@ -202,7 +203,6 @@ func (m *ModuleLoader) Load(path string) (man *Manifest, err error) { // loadModules recursively loads the modules from the given path. func (m *ModuleLoader) loadModules(path string, prefix string) ([]*ManifestModule, error) { - manifestModules := make([]*ManifestModule, 0) module, err := m.loadModuleFromPath(path) if err != nil { @@ -211,6 +211,7 @@ func (m *ModuleLoader) loadModules(path string, prefix string) ([]*ManifestModul numJobs := len(module.ModuleCalls) jobs := make(chan *tfconfig.ModuleCall, numJobs) + manifestModules := make([]*ManifestModule, len(jobs)) for _, moduleCall := range module.ModuleCalls { jobs <- moduleCall } @@ -219,6 +220,8 @@ func (m *ModuleLoader) loadModules(path string, prefix string) ([]*ManifestModul errGroup := &errgroup.Group{} manifestMu := sync.Mutex{} + remoteModuleCounter := metrics.GetCounter("remote_module.count", true) + for i := 0; i < getProcessCount(); i++ { errGroup.Go(func() error { for moduleCall := range jobs { @@ -229,6 +232,7 @@ func (m *ModuleLoader) loadModules(path string, prefix string) ([]*ManifestModul // only include non-local modules in the manifest since we don't want to cache local ones. if !IsLocalModule(metadata.Source) { + remoteModuleCounter.Inc() manifestMu.Lock() manifestModules = append(manifestModules, metadata) manifestMu.Unlock() @@ -336,6 +340,9 @@ func (m *ModuleLoader) loadModule(moduleCall *tfconfig.ModuleCall, parentPath st }, nil } + moduleLoadTimer := metrics.GetTimer("submodule.remote_load.duration", false, source, version).Start() + defer moduleLoadTimer.Stop() + manifestModule, err = m.loadRegistryModule(key, source, version) if err != nil { return nil, err @@ -408,7 +415,7 @@ func (m *ModuleLoader) checkoutPathIfRequired(repoRoot string, dir string) error // RecursivelyAddDirsToSparseCheckout adds the given directories to the sparse-checkout file list. // It then checks any symlinks within the directories and adds them to the sparse-checkout file list as well. func RecursivelyAddDirsToSparseCheckout(repoRoot string, sourceURL string, packageFetcher *PackageFetcher, existingDirs []string, dirs []string, mu *sync.Mutex, logger zerolog.Logger, depth int) error { - var newDirs []string + newDirs := make([]string, 0, len(dirs)) // Sort the existing directories and dirs to be added by length // This ensures that parent directories are added before child directories @@ -724,8 +731,10 @@ func (m *ModuleLoader) loadRegistryModule(key string, source string, version str } manifestModule.Dir = path.Clean(filepath.Join(moduleDownloadDir, submodulePath)) - // lock the module address so that we don't interact with an incomplete download. - unlock := m.sync.Lock(moduleAddr) + // lock the download destination so that we don't interact with an incomplete download. + // we can't use module address as the key here because the module address might be different for the same module, + // e.g. ssh vs https + unlock := m.sync.Lock(dest) defer unlock() lookupResult, err := m.registryLoader.lookupModule(moduleAddr, version) @@ -791,8 +800,9 @@ func (m *ModuleLoader) loadRemoteModule(key string, source string) (*ManifestMod } manifestModule.Dir = path.Clean(filepath.Join(moduleDownloadDir, submodulePath)) - // lock the module address so that we don't interact with an incomplete download. - unlock := m.sync.Lock(moduleAddr) + // lock the download destination so that we don't interact with an incomplete download. + // we can't use module address here as the version may be different + unlock := m.sync.Lock(dest) defer unlock() _, err = os.Stat(dest) @@ -810,7 +820,7 @@ func (m *ModuleLoader) loadRemoteModule(key string, source string) (*ManifestMod } func (m *ModuleLoader) downloadDest(moduleAddr string, version string) string { - hash := fmt.Sprintf("%x", md5.Sum([]byte(moduleAddr+version))) //nolint + hash := fmt.Sprintf("%x", md5.Sum([]byte(moduleAddr+version))) // nolint return filepath.Join(m.downloadDir(), hash) } diff --git a/internal/hcl/modules/loader_test.go b/internal/hcl/modules/loader_test.go index 287d27d4ead..31556686185 100644 --- a/internal/hcl/modules/loader_test.go +++ b/internal/hcl/modules/loader_test.go @@ -24,6 +24,9 @@ type TestLoaderE2EOpts = struct { } func testLoaderE2E(t *testing.T, path string, expectedModules []*ManifestModule, opts TestLoaderE2EOpts) { + + ResetGlobalModuleCache() + if opts.Cleanup { err := os.RemoveAll(filepath.Join(path, config.InfracostDir)) assert.NoError(t, err) diff --git a/internal/hcl/parser.go b/internal/hcl/parser.go index 381476f3dbd..bfa4421bfaa 100644 --- a/internal/hcl/parser.go +++ b/internal/hcl/parser.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/hashicorp/hcl/v2" + "github.com/infracost/infracost/internal/metrics" "github.com/rs/zerolog" "github.com/zclconf/go-cty/cty" "github.com/zclconf/go-cty/cty/gocty" @@ -92,8 +93,8 @@ func (p *Parser) sortVarFilesByPrecedence(paths []string, autoDetected bool) { } func makePathsRelativeToInitial(paths []string, initialPath string) []string { - var filenames []string + filenames := make([]string, 0, len(paths)) for _, name := range paths { if path.IsAbs(name) { filenames = append(filenames, name) @@ -403,7 +404,7 @@ func (p *Parser) DependencyPaths() []string { return nil } - var sortedCalls []string + sortedCalls := make([]string, 0, len(p.moduleCalls)+len(p.tfvarsPaths)+1) for _, call := range p.moduleCalls { relCall := call dep, err := filepath.Rel(p.startingPath, call) @@ -464,9 +465,13 @@ func (p *Parser) ParseDirectory() (m *Module, err error) { p.logger.Debug().Msgf("Beginning parse for directory '%s'...", p.detectedProjectPath) + metrics.GetCounter("hcl_file.count", true).Inc() + // load the initial root directory into a list of hcl files // at this point these files have no schema associated with them. + hclTimer := metrics.GetTimer("hcl_file.load.duration", false, p.detectedProjectPath).Start() files, err := loadDirectory(p.hclParser, p.logger, p.detectedProjectPath, false) + hclTimer.Stop() if err != nil { return m, err } @@ -481,8 +486,12 @@ func (p *Parser) ParseDirectory() (m *Module, err error) { return m, errors.New("No valid terraform files found given path, try a different directory") } + metrics.GetCounter("block.count", true).Add(len(blocks)) + p.logger.Debug().Msg("Loading TFVars...") + varLoadTimer := metrics.GetTimer("var.load.duration", false, p.detectedProjectPath).Start() inputVars, err := p.loadVars(blocks, p.tfvarsPaths) + varLoadTimer.Stop() if err != nil { return m, err } @@ -521,6 +530,9 @@ func (p *Parser) ParseDirectory() (m *Module, err error) { var root *Module + evaluationTimer := metrics.GetTimer("eval.duration", false, p.detectedProjectPath).Start() + defer evaluationTimer.Stop() + // Graph evaluation if evaluator.isGraph { logging.Logger.Debug().Msg("Building project with experimental graph runner") diff --git a/internal/hcl/project_locator.go b/internal/hcl/project_locator.go index f482526dfae..8917991b0c2 100644 --- a/internal/hcl/project_locator.go +++ b/internal/hcl/project_locator.go @@ -100,7 +100,7 @@ func CreateEnvFileMatcher(names []string, extensions []string) *EnvFileMatcher { return len(extensions[i]) > len(extensions[j]) }) - var envNames []string + envNames := make([]string, 0, len(names)) var wildcards []string for _, name := range names { // envNames can contain wildcards, we need to handle them separately. e.g: dev-* @@ -130,7 +130,7 @@ func CreateEnvFileMatcher(names []string, extensions []string) *EnvFileMatcher { } func createWildcardGlobPaths(dirs []string) []string { - var paths []string + paths := make([]string, 0, len(dirs)*2) for _, dir := range dirs { paths = append(paths, path.Join(dir, "**")) } @@ -494,7 +494,7 @@ func CreateTreeNode(basePath string, paths []RootPath, varFiles map[string][]Roo root.AddPath(path) } - var varFilesSorted []string + varFilesSorted := make([]string, 0, len(varFiles)) for dir := range varFiles { varFilesSorted = append(varFilesSorted, dir) } @@ -1116,13 +1116,13 @@ func (r *RootPath) EnvGroupings() []VarFileGrouping { } } - var envNames []string + envNames := make([]string, 0, len(varFileGrouping)) for env := range varFileGrouping { envNames = append(envNames, env) } sort.Strings(envNames) - var varEnvs []VarFileGrouping + varEnvs := make([]VarFileGrouping, 0, len(envNames)) for _, env := range envNames { varEnvs = append(varEnvs, VarFileGrouping{ Name: env, @@ -1752,7 +1752,7 @@ func (p *ProjectLocator) findTerragruntDirs(fullPath string) { // removeParentTerragruntFiles removes any parent Terragrunt config files from // the list of discovered Terragrunt configuration files. func (p *ProjectLocator) removeParentTerragruntFiles(startingPath string, files []string) []string { - var paths []RootPath + paths := make([]RootPath, 0, len(files)) nameMap := make(map[string]string) for _, file := range files { dir := filepath.Dir(file) diff --git a/internal/metrics/counter.go b/internal/metrics/counter.go new file mode 100644 index 00000000000..6738ec23b3b --- /dev/null +++ b/internal/metrics/counter.go @@ -0,0 +1,42 @@ +package metrics + +import "sync" + +type Counter struct { + value int + mu sync.Mutex +} + +func (c *Counter) Result() Result { + c.mu.Lock() + defer c.mu.Unlock() + return Result{ + Value: c.value, + } +} + +func GetCounter(name string, reuse bool, context ...string) *Counter { + if reuse { + existing := coreRegistry.Find(name, TypeCounter) + if existing != nil { + return existing.(*Counter) + } + } + c := &Counter{} + coreRegistry.Add(name, TypeCounter, c, context...) + return c +} + +func (c *Counter) Inc() *Counter { + c.mu.Lock() + defer c.mu.Unlock() + c.value++ + return c +} + +func (c *Counter) Add(value int) *Counter { + c.mu.Lock() + defer c.mu.Unlock() + c.value += value + return c +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 00000000000..5a8aee7e229 --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,113 @@ +package metrics + +import ( + "encoding/json" + "os" + "sync" +) + +type Type string + +const ( + TypeCounter Type = "counter" + TypeTimer Type = "timer" + TypeSetting Type = "setting" +) + +type Metric interface { + Result() Result +} + +type Result struct { + Unit string + Value interface{} +} + +type registeredMetric struct { + Name string + Key int + Type Type + Context []string + Metric Metric +} + +type registry struct { + mu sync.RWMutex + metrics []registeredMetric +} + +var coreRegistry = ®istry{} + +func (r *registry) Add(name string, t Type, m Metric, context ...string) { + r.mu.Lock() + defer r.mu.Unlock() + highestKey := -1 + for _, m := range r.metrics { + if m.Name == name && m.Type == t && m.Key > highestKey { + highestKey = m.Key + } + } + r.metrics = append(r.metrics, registeredMetric{ + Name: name, + Type: t, + Metric: m, + Key: highestKey + 1, + Context: context, + }) +} + +func (r *registry) Find(name string, t Type) Metric { + r.mu.RLock() + defer r.mu.RUnlock() + for _, m := range r.metrics { + if m.Name == name && m.Type == t { + return m.Metric + } + } + return nil +} + +type Output []OutputMetric + +type OutputMetric struct { + Name string `json:"name"` + Unit string `json:"unit,omitempty"` + Values []OutputValue `json:"values,omitempty"` +} + +type OutputValue struct { + Context []string `json:"context,omitempty"` + Value interface{} `json:"value"` +} + +func WriteMetrics(path string) error { + coreRegistry.mu.RLock() + defer coreRegistry.mu.RUnlock() + + outputMap := map[string]*OutputMetric{} + + for _, m := range coreRegistry.metrics { + + result := m.Metric.Result() + + metric := outputMap[m.Name] + if metric == nil { + metric = &OutputMetric{ + Name: m.Name, + Unit: result.Unit, + } + outputMap[m.Name] = metric + } + + metric.Values = append(metric.Values, OutputValue{ + Context: m.Context, + Value: result.Value, + }) + } + + data, err := json.Marshal(outputMap) + if err != nil { + return err + } + return os.WriteFile(path, data, 0600) +} diff --git a/internal/metrics/setting.go b/internal/metrics/setting.go new file mode 100644 index 00000000000..47adb261ec2 --- /dev/null +++ b/internal/metrics/setting.go @@ -0,0 +1,29 @@ +package metrics + +import "sync" + +type Setting struct { + value interface{} + mu sync.Mutex +} + +func (s *Setting) Result() Result { + s.mu.Lock() + defer s.mu.Unlock() + return Result{ + Value: s.value, + } +} + +func GetSetting(name string, context ...string) *Setting { + c := &Setting{} + coreRegistry.Add(name, TypeSetting, c, context...) + return c +} + +func (s *Setting) Set(value any) *Setting { + s.mu.Lock() + defer s.mu.Unlock() + s.value = value + return s +} diff --git a/internal/metrics/timer.go b/internal/metrics/timer.go new file mode 100644 index 00000000000..79ca339f4ff --- /dev/null +++ b/internal/metrics/timer.go @@ -0,0 +1,64 @@ +package metrics + +import ( + "sync" + "time" +) + +type Timer struct { + startTime time.Time + elapsedTime time.Duration + running bool + mu sync.Mutex +} + +func (t *Timer) Result() Result { + t.mu.Lock() + defer t.mu.Unlock() + elapsed := t.elapsedTime.Milliseconds() + if t.running { + elapsed += time.Since(t.startTime).Milliseconds() + } + return Result{ + Unit: "ms", + Value: elapsed, + } +} + +func GetTimer(name string, reuse bool, context ...string) *Timer { + if reuse { + existing := coreRegistry.Find(name, TypeTimer) + if existing != nil { + return existing.(*Timer) + } + } + t := &Timer{} + coreRegistry.Add(name, TypeTimer, t, context...) + return t +} + +func (t *Timer) Start() *Timer { + t.mu.Lock() + defer t.mu.Unlock() + + if t.running { + return t + } + + t.startTime = time.Now() + t.running = true + return t +} + +func (t *Timer) Stop() *Timer { + t.mu.Lock() + defer t.mu.Unlock() + + if !t.running { + return t + } + + t.elapsedTime += time.Since(t.startTime) + t.running = false + return t +} diff --git a/internal/output/combined.go b/internal/output/combined.go index 578ed107832..610f987df55 100644 --- a/internal/output/combined.go +++ b/internal/output/combined.go @@ -11,6 +11,7 @@ import ( "time" "github.com/Rhymond/go-money" + "github.com/infracost/infracost/internal/metrics" "github.com/mitchellh/go-homedir" "github.com/pkg/errors" "github.com/shopspring/decimal" @@ -36,6 +37,8 @@ type ReportInput struct { // Load reads the file at the location p and the file body into a Root struct. Load naively // validates that the Infracost JSON body is valid by checking the that the version attribute is within a supported range. func Load(p string) (Root, error) { + timer := metrics.GetTimer("output.load.duration", false).Start() + defer timer.Stop() var out Root _, err := os.Stat(p) if errors.Is(err, os.ErrNotExist) { @@ -118,6 +121,8 @@ func LoadPaths(paths []string) ([]ReportInput, error) { // in the prior Root. If we can't find a matching project then we assume that the project // has been newly created and will show a 100% increase in the output Root. func CompareTo(c *config.Config, current, prior Root) (Root, error) { + compareTimer := metrics.GetTimer("output.compare.duration", false).Start() + defer compareTimer.Stop() currentProjectLabels := make(map[string]bool, len(current.Projects)) for _, p := range current.Projects { currentProjectLabels[p.LabelWithMetadata()] = true diff --git a/internal/output/table.go b/internal/output/table.go index 0eb6e15ada3..d9f8057073f 100644 --- a/internal/output/table.go +++ b/internal/output/table.go @@ -370,7 +370,7 @@ func buildActualCostRows(t table.Writer, currency string, actualCosts []ActualCo } func filterZeroValComponents(costComponents []CostComponent, resourceName string) []CostComponent { - var filteredComponents []CostComponent + filteredComponents := make([]CostComponent, 0, len(costComponents)) for _, c := range costComponents { if c.MonthlyQuantity != nil && c.MonthlyQuantity.IsZero() { logging.Logger.Debug().Msgf("Hiding cost with no usage: %s '%s'", resourceName, c.Name) @@ -383,7 +383,7 @@ func filterZeroValComponents(costComponents []CostComponent, resourceName string } func filterZeroValResources(resources []Resource, resourceName string) []Resource { - var filteredResources []Resource + filteredResources := make([]Resource, 0, len(resources)) for _, r := range resources { filteredComponents := filterZeroValComponents(r.CostComponents, fmt.Sprintf("%s.%s", resourceName, r.Name)) filteredSubResources := filterZeroValResources(r.SubResources, fmt.Sprintf("%s.%s", resourceName, r.Name)) diff --git a/internal/prices/prices.go b/internal/prices/prices.go index b267b625cca..b82f5e960ba 100644 --- a/internal/prices/prices.go +++ b/internal/prices/prices.go @@ -20,7 +20,7 @@ import ( ) var ( - batchSize = 5 + batchSize = 1000 ) // notFoundData represents a single price not found entry @@ -48,7 +48,7 @@ func NewPriceFetcher(ctx *config.RunContext, warnOnPriceErrors bool) *PriceFetch components: make(map[string]int), mux: &sync.RWMutex{}, runCtx: ctx, - client: apiclient.NewPricingAPIClient(ctx), + client: apiclient.GetPricingAPIClient(ctx), warnOnPriceErrors: warnOnPriceErrors, } } @@ -144,7 +144,7 @@ func (p *PriceFetcher) LogWarnings() { return } - var data []*notFoundData + data := make([]*notFoundData, 0, len(p.resources)) for _, v := range p.resources { data = append(data, v) } @@ -201,7 +201,7 @@ func (p *PriceFetcher) PopulatePrices(project *schema.Project) error { // getPricesConcurrent gets the prices of all resources concurrently. // Concurrency level is calculated using the following formula: -// max(min(4, numCPU * 4), 16) +// min(max(4, numCPU * 4), 16) func (p *PriceFetcher) getPricesConcurrent(resources []*schema.Resource) error { // Set the number of workers numWorkers := 4 @@ -234,6 +234,8 @@ func (p *PriceFetcher) getPricesConcurrent(resources []*schema.Resource) error { jobs <- r } + close(jobs) + // Get the result of the jobs for i := 0; i < numJobs; i++ { err := <-resultErrors @@ -241,6 +243,9 @@ func (p *PriceFetcher) getPricesConcurrent(resources []*schema.Resource) error { return err } } + + close(resultErrors) + return nil } diff --git a/internal/providers/cloudformation/parser.go b/internal/providers/cloudformation/parser.go index 75d7163a1a8..fc1ae3b41f0 100644 --- a/internal/providers/cloudformation/parser.go +++ b/internal/providers/cloudformation/parser.go @@ -91,7 +91,7 @@ func (p *Parser) createResource(d *schema.ResourceData, u *schema.UsageData) par } func (p *Parser) parseTemplate(t *cloudformation.Template, usage schema.UsageMap) []parsedResource { - var resources []parsedResource + resources := make([]parsedResource, 0, len(t.Resources)) for name, d := range t.Resources { resourceData := schema.NewCFResourceData(d.AWSCloudFormationType(), "aws", name, nil, d) diff --git a/internal/providers/detect.go b/internal/providers/detect.go index 069db703a9c..4455f3298c5 100644 --- a/internal/providers/detect.go +++ b/internal/providers/detect.go @@ -9,7 +9,6 @@ import ( "sync" "github.com/awslabs/goformation/v7" - "github.com/infracost/infracost/internal/config" "github.com/infracost/infracost/internal/hcl" "github.com/infracost/infracost/internal/logging" diff --git a/internal/providers/terraform/hcl_provider.go b/internal/providers/terraform/hcl_provider.go index c0d2309c9ec..296f750ee60 100644 --- a/internal/providers/terraform/hcl_provider.go +++ b/internal/providers/terraform/hcl_provider.go @@ -15,6 +15,7 @@ import ( "sort" "strings" + "github.com/infracost/infracost/internal/metrics" "github.com/infracost/infracost/internal/schema" jsoniter "github.com/json-iterator/go" @@ -262,13 +263,21 @@ func (p *HCLProvider) AddMetadata(metadata *schema.ProjectMetadata) { // The PlanJSONProvider uses this shallow representation to actually load Infracost resources. func (p *HCLProvider) LoadResources(usage schema.UsageMap) ([]*schema.Project, error) { + loadResourcesTimer := metrics.GetTimer("hcl.LoadResources", false, p.ctx.ProjectConfig.Path).Start() + defer loadResourcesTimer.Stop() + + loadPlanTimer := metrics.GetTimer("hcl.LoadPlanJSON", false, p.ctx.ProjectConfig.Path).Start() j := p.LoadPlanJSON() + loadPlanTimer.Stop() if j.Error != nil { return []*schema.Project{p.newProject(j)}, nil } project := p.newProject(j) + + parseJSONTimer := metrics.GetTimer("hcl.ParseJSON", false, p.ctx.ProjectConfig.Path).Start() parsedConf, err := p.planJSONParser.parseJSON(j.JSON, usage) + parseJSONTimer.Stop() if err != nil { project.Metadata.AddError(schema.NewDiagJSONParsingFailure(err)) @@ -286,7 +295,9 @@ func (p *HCLProvider) LoadResources(usage schema.UsageMap) ([]*schema.Project, e project.PartialResources = parsedConf.CurrentResources if p.policyClient != nil { + uploadPolicyDataTimer := metrics.GetTimer("hcl.UploadPolicyData", false, p.ctx.ProjectConfig.Path).Start() err := p.policyClient.UploadPolicyData(project, parsedConf.CurrentResourceDatas, parsedConf.PastResourceDatas) + uploadPolicyDataTimer.Stop() if err != nil { p.logger.Err(err).Msgf("failed to upload policy data %s", project.Name) } @@ -373,10 +384,16 @@ func (p *HCLProvider) LoadPlanJSON() HCLProject { // found Terraform project. This can be used to fetch raw information like // outputs, vars, resources, e.t.c. func (p *HCLProvider) Module() HCLProject { + + metrics.GetCounter("root_module.count", true).Inc() + if p.cache != nil { return *p.cache } + parseTimer := metrics.GetTimer("root_module.parse.duration", false, p.Context().ProjectConfig.Path).Start() + defer parseTimer.Stop() + module, modErr := p.Parser.ParseDirectory() var v *clierror.PanicError if errors.As(modErr, &v) { @@ -452,6 +469,9 @@ func (p *HCLProvider) modulesToPlanJSON(rootModule *hcl.Module) ([]byte, error) } func (p *HCLProvider) marshalModule(module *hcl.Module) ModuleOut { + + metrics.GetCounter("module.count", true).Inc() + moduleConfig := ModuleConfig{ ModuleCalls: map[string]ModuleCall{}, } @@ -476,6 +496,8 @@ func (p *HCLProvider) marshalModule(module *hcl.Module) ModuleOut { if block.Type() == "resource" { out := p.getResourceOutput(block, module.SourceURL) + metrics.GetCounter("resource.count", true).Inc() + if _, ok := configResources[out.Configuration.Address]; !ok { moduleConfig.Resources = append(moduleConfig.Resources, out.Configuration) diff --git a/internal/providers/terraform/parser.go b/internal/providers/terraform/parser.go index b2120569502..5a6e687a31e 100644 --- a/internal/providers/terraform/parser.go +++ b/internal/providers/terraform/parser.go @@ -119,8 +119,7 @@ func (p *Parser) createParsedResource(d *schema.ResourceData, u *schema.UsageDat } func (p *Parser) parseJSONResources(parsePrior bool, baseResources []parsedResource, usage schema.UsageMap, confLoader *ConfLoader, parsed, providerConf, vars gjson.Result) []parsedResource { - var resources []parsedResource - resources = append(resources, baseResources...) + var vals gjson.Result isState := false @@ -137,6 +136,9 @@ func (p *Parser) parseJSONResources(parsePrior bool, baseResources []parsedResou resData := p.parseResourceData(isState, confLoader, providerConf, vals, vars) + resources := make([]parsedResource, len(baseResources), len(resData)+len(baseResources)) + copy(resources, baseResources) + p.parseReferences(resData, confLoader) p.parseTags(resData, confLoader, providerConf) @@ -294,7 +296,7 @@ func collectModulesSourceUrls(moduleCalls []gjson.Result) []string { return nil } - var urls []string + urls := make([]string, 0, len(remoteUrls)) for source := range remoteUrls { // Perf/memory leak: Copy gjson string slices that may be returned so we don't prevent // the entire underlying parsed json from being garbage collected. @@ -306,7 +308,6 @@ func collectModulesSourceUrls(moduleCalls []gjson.Result) []string { } func parseProviderConfig(providerConf gjson.Result) []schema.ProviderMetadata { - var metadatas []schema.ProviderMetadata confMap := providerConf.Map() @@ -317,6 +318,8 @@ func parseProviderConfig(providerConf gjson.Result) []schema.ProviderMetadata { } sort.Strings(keys) + metadatas := make([]schema.ProviderMetadata, 0, len(keys)) + for _, k := range keys { conf := confMap[k] md := schema.ProviderMetadata{ @@ -532,13 +535,14 @@ func parseGoogleDefaultTags(providerConf, resConf gjson.Result) (map[string]stri for k, v := range providerConf.Get(fmt.Sprintf("%s.expressions.default_labels.constant_value", gjsonEscape(providerKey))).Map() { defaultTags[k] = v.String() } - var missingAttrsCausingUnknownKeys []string - for _, address := range providerConf.Get( + missingAttributes := providerConf.Get( fmt.Sprintf( "%s.expressions.default_labels.missing_attributes_causing_unknown_keys", gjsonEscape(providerKey), ), - ).Array() { + ).Array() + missingAttrsCausingUnknownKeys := make([]string, 0, len(missingAttributes)) + for _, address := range missingAttributes { if address.String() == "" { continue } diff --git a/internal/providers/terraform/terragrunt_hcl_provider.go b/internal/providers/terraform/terragrunt_hcl_provider.go index 84dc0fb42a5..cee93d0bfe5 100644 --- a/internal/providers/terraform/terragrunt_hcl_provider.go +++ b/internal/providers/terraform/terragrunt_hcl_provider.go @@ -32,6 +32,7 @@ import ( hcl2 "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/gohcl" "github.com/hashicorp/hcl/v2/hclparse" + "github.com/infracost/infracost/internal/metrics" "github.com/rs/zerolog" "github.com/sirupsen/logrus" "github.com/zclconf/go-cty/cty" @@ -266,6 +267,10 @@ func (i *terragruntWorkingDirInfo) addWarning(pd *schema.ProjectDiag) { // LoadResources finds any Terragrunt projects, prepares them by downloading any required source files, then // process each with an HCLProvider. func (p *TerragruntHCLProvider) LoadResources(usage schema.UsageMap) ([]*schema.Project, error) { + + loadResourcesTimer := metrics.GetTimer("terragrunt.LoadResources", false, p.ctx.ProjectConfig.Path).Start() + defer loadResourcesTimer.Stop() + dirs, err := p.prepWorkingDirs() if err != nil { return nil, err diff --git a/internal/resources/google/artifact_registry_repository.go b/internal/resources/google/artifact_registry_repository.go index c6652192272..cbfb7e3ea17 100644 --- a/internal/resources/google/artifact_registry_repository.go +++ b/internal/resources/google/artifact_registry_repository.go @@ -192,7 +192,7 @@ func (r *ArtifactRegistryRepository) toEgressFilters() []artifactRegistryEgressF values := r.MonthlyEgressDataTransferGB.Values() - var data []artifactRegistryEgressFilters + data := make([]artifactRegistryEgressFilters, 0, len(values)) transferMap := make(map[string]int) for _, region := range values { diff --git a/internal/schema/project.go b/internal/schema/project.go index b0470ced3d3..0c5c20bcd11 100644 --- a/internal/schema/project.go +++ b/internal/schema/project.go @@ -413,7 +413,7 @@ func (p *Project) AllResources() []*Resource { } } - var resources []*Resource + resources := make([]*Resource, 0, len(m)) for r := range m { resources = append(resources, r) } @@ -434,7 +434,7 @@ func (p *Project) AllPartialResources() []*PartialResource { } } - var resources []*PartialResource + resources := make([]*PartialResource, 0, len(m)) for r := range m { resources = append(resources, r) } diff --git a/internal/usage/usage_file.go b/internal/usage/usage_file.go index b52499bbaae..631312f5f16 100644 --- a/internal/usage/usage_file.go +++ b/internal/usage/usage_file.go @@ -7,6 +7,7 @@ import ( "sort" "strings" + "github.com/infracost/infracost/internal/metrics" "github.com/pkg/errors" "golang.org/x/mod/semver" yamlv3 "gopkg.in/yaml.v3" @@ -48,6 +49,8 @@ func CreateUsageFile(path string) error { } func LoadUsageFile(path string) (*UsageFile, error) { + loadUsageTimer := metrics.GetTimer("parallel_runner.load_usage.duration", false, path).Start() + defer loadUsageTimer.Stop() blankUsage := NewBlankUsageFile() if _, err := os.Stat(path); os.IsNotExist(err) { logging.Logger.Debug().Msg("Specified usage file does not exist. Using a blank file") diff --git a/internal/vcs/metadata.go b/internal/vcs/metadata.go index fb923eb869c..d404b57a1ae 100644 --- a/internal/vcs/metadata.go +++ b/internal/vcs/metadata.go @@ -600,7 +600,7 @@ func (f *metadataFetcher) getFileChanges(path string, r *git.Repository, current changedMap[change.To.Name] = struct{}{} } - var changedFiles []string + changedFiles := make([]string, 0, len(changedMap)) for name := range changedMap { if name == "" { continue From 1463716101e0d47e211bb155514f067eed75ee0e Mon Sep 17 00:00:00 2001 From: Hugo Rut Date: Thu, 12 Dec 2024 11:05:28 +0100 Subject: [PATCH 56/81] feat: add prefer_folder_name_for_env autodetect config (#3269) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds new configuration field which allows users to specify how envs are inferred. Setting `prefer_folder_name_for_env: true` means that the envs are inferred from folder names rather than the terraform vars contained in that folder. This is to solve the case where customers have envs defined by folder names but also with coliding tfvar names e.g: ``` . └── infra ├── envs │ ├── stg │ │ └── prod.tfvars │ └── qa │ └── prod.tfvars └── main.tf ``` This normally occurs if they have strange environment setup or they have mis spelled or copied a tfvar file. --- .../prefer_folder_name/expected.golden | 20 ++++++++++ .../prefer_folder_name/infracost.yml.tmpl | 8 ++++ .../generate/prefer_folder_name/tree.txt | 8 ++++ internal/config/config.go | 13 +++++++ internal/hcl/project_locator.go | 38 ++++++++++++++----- internal/providers/detect.go | 1 + 6 files changed, 78 insertions(+), 10 deletions(-) create mode 100755 cmd/infracost/testdata/generate/prefer_folder_name/expected.golden create mode 100644 cmd/infracost/testdata/generate/prefer_folder_name/infracost.yml.tmpl create mode 100644 cmd/infracost/testdata/generate/prefer_folder_name/tree.txt diff --git a/cmd/infracost/testdata/generate/prefer_folder_name/expected.golden b/cmd/infracost/testdata/generate/prefer_folder_name/expected.golden new file mode 100755 index 00000000000..eacbfc47434 --- /dev/null +++ b/cmd/infracost/testdata/generate/prefer_folder_name/expected.golden @@ -0,0 +1,20 @@ +version: 0.1 + +projects: + - path: infra + name: infra-qa + terraform_var_files: + - envs/qa/prod.tfvars + skip_autodetect: true + dependency_paths: + - infra/** + - infra/envs/qa/prod.tfvars + - path: infra + name: infra-stg + terraform_var_files: + - envs/stg/dev.tfvars + skip_autodetect: true + dependency_paths: + - infra/** + - infra/envs/stg/dev.tfvars + diff --git a/cmd/infracost/testdata/generate/prefer_folder_name/infracost.yml.tmpl b/cmd/infracost/testdata/generate/prefer_folder_name/infracost.yml.tmpl new file mode 100644 index 00000000000..8724b3e3735 --- /dev/null +++ b/cmd/infracost/testdata/generate/prefer_folder_name/infracost.yml.tmpl @@ -0,0 +1,8 @@ +version: 0.1 +autodetect: + env_names: + - qa + - dev + - prod + - stg + prefer_folder_name_for_env: true diff --git a/cmd/infracost/testdata/generate/prefer_folder_name/tree.txt b/cmd/infracost/testdata/generate/prefer_folder_name/tree.txt new file mode 100644 index 00000000000..be1c0533d6f --- /dev/null +++ b/cmd/infracost/testdata/generate/prefer_folder_name/tree.txt @@ -0,0 +1,8 @@ +. +└── infra + ├── envs + │ ├── stg + │ │ └── dev.tfvars + │ └── qa + │ └── prod.tfvars + └── main.tf diff --git a/internal/config/config.go b/internal/config/config.go index cd77d872c0e..7be0397b032 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -43,6 +43,19 @@ type AutodetectConfig struct { // var files. This is useful when there are non-standard terraform var file // names which use different extensions. TerraformVarFileExtensions []string `yaml:"terraform_var_file_extensions,omitempty" ignored:"true"` + // PreferFolderNameForEnv tells the autodetect to prefer the folder name over the + // over a env specified in a tfvars file. For example, given the following + // folder structure: + // + // . + // ├── qa + // │ └── dev.tfvars + // └── staging + // └── prod.tfvars + // + // If PreferFolderNameForEnv is true, then the autodetect will group the projects + // by the folder name so the projects will be named "qa" and "staging". + PreferFolderNameForEnv bool `yaml:"prefer_folder_name_for_env,omitempty" ignored:"true"` } type PathOverride struct { diff --git a/internal/hcl/project_locator.go b/internal/hcl/project_locator.go index 8917991b0c2..b00841dede0 100644 --- a/internal/hcl/project_locator.go +++ b/internal/hcl/project_locator.go @@ -314,6 +314,7 @@ type ProjectLocator struct { hclParser *hclparse.Parser hasCustomEnvExt bool workingDirectory string + preferFolderNameForEnv bool } // ProjectLocatorConfig provides configuration options on how the locator functions. @@ -330,6 +331,7 @@ type ProjectLocatorConfig struct { ForceProjectType string TerraformVarFileExtensions []string WorkingDirectory string + PreferFolderNameForEnv bool } type PathOverrideConfig struct { @@ -410,6 +412,7 @@ func NewProjectLocator(logger zerolog.Logger, config *ProjectLocatorConfig) *Pro hclParser: hclparse.NewParser(), hasCustomEnvExt: len(config.TerraformVarFileExtensions) > 0, workingDirectory: config.WorkingDirectory, + preferFolderNameForEnv: config.PreferFolderNameForEnv, } } @@ -481,7 +484,7 @@ type VarFiles struct { // CreateTreeNode creates a tree of Terraform projects and directories that // contain var files. -func CreateTreeNode(basePath string, paths []RootPath, varFiles map[string][]RootPathVarFile, e *EnvFileMatcher) *TreeNode { +func CreateTreeNode(basePath string, paths []RootPath, varFiles map[string][]RootPathVarFile, e *EnvFileMatcher, preferFolderNameForEnv bool) *TreeNode { root := &TreeNode{ Name: "root", } @@ -506,14 +509,14 @@ func CreateTreeNode(basePath string, paths []RootPath, varFiles map[string][]Roo root.AddTerraformVarFiles(basePath, dir, varFiles[dir]) } - buildVarFileEnvNames(root, e) + buildVarFileEnvNames(root, e, preferFolderNameForEnv) return root } // buildVarFileEnvNames builds the EnvName field for each var file. Var names // can be inferred from the filename or from parent directories, but only if // the parent directories don't contain any Terraform projects. -func buildVarFileEnvNames(root *TreeNode, e *EnvFileMatcher) { +func buildVarFileEnvNames(root *TreeNode, e *EnvFileMatcher, preferFolderNameForEnv bool) { root.PostOrder(func(t *TreeNode) { if t.TerraformVarFiles == nil { return @@ -543,12 +546,27 @@ func buildVarFileEnvNames(root *TreeNode, e *EnvFileMatcher) { } for i, f := range t.TerraformVarFiles.Files { - namesToSearch := append([]string{f.Name}, possibleEnvNames...) var envName string - for _, search := range namesToSearch { - if e.IsEnvName(search) { - envName = search - break + if preferFolderNameForEnv { + // First try to find env name from parent directories + for _, search := range possibleEnvNames { + if e.IsEnvName(search) { + envName = search + break + } + } + + // If no env name found in parent directories, fallback to file name + if envName == "" && e.IsEnvName(f.Name) { + envName = f.Name + } + } else { + namesToSearch := append([]string{f.Name}, possibleEnvNames...) + for _, search := range namesToSearch { + if e.IsEnvName(search) { + envName = search + break + } } } @@ -1260,7 +1278,7 @@ func (p *ProjectLocator) FindRootModules(startingPath string) []RootPath { delete(p.discoveredVarFiles, dir.DetectedPath) } - node := CreateTreeNode(startingPath, projects, p.discoveredVarFiles, p.envMatcher) + node := CreateTreeNode(startingPath, projects, p.discoveredVarFiles, p.envMatcher, p.preferFolderNameForEnv) node.AssociateChildVarFiles() node.AssociateSiblingVarFiles() node.AssociateParentVarFiles() @@ -1765,7 +1783,7 @@ func (p *ProjectLocator) removeParentTerragruntFiles(startingPath string, files } var projects []string - root := CreateTreeNode(startingPath, paths, nil, nil) + root := CreateTreeNode(startingPath, paths, nil, nil, p.preferFolderNameForEnv) // We need to slightly modify the TreeNode behaviour so that it works for this Terragrunt // use case. For this case if there is a detected Terragrunt config file in the root of the diff --git a/internal/providers/detect.go b/internal/providers/detect.go index 4455f3298c5..84b30d7209f 100644 --- a/internal/providers/detect.go +++ b/internal/providers/detect.go @@ -76,6 +76,7 @@ func Detect(ctx *config.RunContext, project *config.Project, includePastResource MaxSearchDepth: ctx.Config.Autodetect.MaxSearchDepth, ForceProjectType: ctx.Config.Autodetect.ForceProjectType, TerraformVarFileExtensions: ctx.Config.Autodetect.TerraformVarFileExtensions, + PreferFolderNameForEnv: ctx.Config.Autodetect.PreferFolderNameForEnv, } // if the config file path is set, we should set the project locator to use the From b60e413384614bb8fa46f2e7f062b399c564d3c3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 09:31:58 +0000 Subject: [PATCH 57/81] chore(deps): bump golang.org/x/crypto from 0.23.0 to 0.31.0 Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.23.0 to 0.31.0. - [Commits](https://github.com/golang/crypto/compare/v0.23.0...v0.31.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 14 +++++++------- go.sum | 28 ++++++++++++++-------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index bd60bcf78b4..ddf51bc690d 100644 --- a/go.mod +++ b/go.mod @@ -38,8 +38,8 @@ require ( github.com/stretchr/testify v1.8.4 github.com/tidwall/gjson v1.17.0 github.com/zclconf/go-cty v1.14.1 - golang.org/x/crypto v0.23.0 // indirect - golang.org/x/mod v0.14.0 + golang.org/x/crypto v0.31.0 // indirect + golang.org/x/mod v0.17.0 gopkg.in/go-playground/assert.v1 v1.2.1 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 @@ -48,7 +48,7 @@ require ( require ( github.com/aws/aws-sdk-go-v2/service/eks v1.27.0 github.com/hashicorp/terraform-config-inspect v0.0.0-20210625153042-09f34846faab - golang.org/x/sys v0.21.0 // indirect + golang.org/x/sys v0.28.0 // indirect ) require ( @@ -79,8 +79,8 @@ require ( github.com/slack-go/slack v0.12.3 github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect - golang.org/x/text v0.15.0 - golang.org/x/tools v0.16.1 // indirect + golang.org/x/text v0.21.0 + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect ) @@ -88,7 +88,7 @@ require ( github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect github.com/gorilla/websocket v1.4.2 // indirect - golang.org/x/sync v0.7.0 + golang.org/x/sync v0.10.0 ) require ( @@ -222,7 +222,7 @@ require ( github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect go.mozilla.org/gopgagent v0.0.0-20170926210634-4d7ea76ff71a // indirect go.mozilla.org/sops/v3 v3.7.3 // indirect - golang.org/x/term v0.21.0 // indirect + golang.org/x/term v0.27.0 // indirect golang.org/x/time v0.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.66.4 // indirect diff --git a/go.sum b/go.sum index 655be61008e..55a54c14f26 100644 --- a/go.sum +++ b/go.sum @@ -1247,8 +1247,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1288,8 +1288,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1397,8 +1397,8 @@ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1491,8 +1491,8 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1501,8 +1501,8 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1517,8 +1517,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1584,8 +1584,8 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= -golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From be06c2fa3f67a7d96e6c589f00bf372e196dd99b Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Tue, 17 Dec 2024 13:37:49 +0000 Subject: [PATCH 58/81] chore: update golden tests --- .../breakdown_terragrunt_with_remote_source.golden | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/infracost/testdata/breakdown_terragrunt_with_remote_source/breakdown_terragrunt_with_remote_source.golden b/cmd/infracost/testdata/breakdown_terragrunt_with_remote_source/breakdown_terragrunt_with_remote_source.golden index e0b54e2835b..f1f89b464dc 100644 --- a/cmd/infracost/testdata/breakdown_terragrunt_with_remote_source/breakdown_terragrunt_with_remote_source.golden +++ b/cmd/infracost/testdata/breakdown_terragrunt_with_remote_source/breakdown_terragrunt_with_remote_source.golden @@ -159,9 +159,9 @@ Module path: REPLACED_PROJECT_PATH/testdata/breakdown_terragrunt_with_remote_sou *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. ────────────────────────────────── -115 cloud resources were detected: +117 cloud resources were detected: ∙ 21 were estimated -∙ 93 were free +∙ 95 were free ∙ 1 is not supported yet, rerun with --show-skipped to see details ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ From fa5ec6bdff8c85dc2d13086082aa849e88144dca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 13:54:36 +0000 Subject: [PATCH 59/81] chore(deps): bump github.com/golang-jwt/jwt/v4 from 4.5.0 to 4.5.1 Bumps [github.com/golang-jwt/jwt/v4](https://github.com/golang-jwt/jwt) from 4.5.0 to 4.5.1. - [Release notes](https://github.com/golang-jwt/jwt/releases) - [Changelog](https://github.com/golang-jwt/jwt/blob/main/VERSION_HISTORY.md) - [Commits](https://github.com/golang-jwt/jwt/compare/v4.5.0...v4.5.1) --- updated-dependencies: - dependency-name: github.com/golang-jwt/jwt/v4 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ddf51bc690d..a0085504e6d 100644 --- a/go.mod +++ b/go.mod @@ -165,7 +165,7 @@ require ( github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-jose/go-jose/v4 v4.0.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.0 // indirect + github.com/golang-jwt/jwt/v4 v4.5.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-github/v35 v35.3.0 // indirect diff --git a/go.sum b/go.sum index 55a54c14f26..cb072ab5b72 100644 --- a/go.sum +++ b/go.sum @@ -564,8 +564,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= +github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= From c9d92fe2c47b23dc81ea159dae2a846edf128109 Mon Sep 17 00:00:00 2001 From: Owen Date: Tue, 17 Dec 2024 15:20:32 +0000 Subject: [PATCH 60/81] fix: handle S3 bucket module ID issues (#3276) * fix: handle S3 bucket module ID issues Resolves an issue where a known module (https://github.com/terraform-aws-modules/terraform-aws-s3-bucket) returns a coalesced ID starting with the id of the bucket policy. This does not work with our generated IDs and can result in cases where a lifecycle configuration that is referencing the `s3_bucket_id` output on the module generated bucket. This fix is specifically very targetted to `aws_s3_bucket_lifecycle_configuration`, we should really only add the rewrites when we know there is an issue * test: fix tests * test: update the aws goldens --- .../dynamodb_table_china_test.golden | 26 +-- .../lambda_function_china_test.golden | 14 +- .../rds_cluster_china_test.golden | 4 +- .../s3_bucket_china_test.golden | 148 +++++++++--------- internal/providers/terraform/parser.go | 44 ++++++ internal/providers/terraform/parser_test.go | 44 ++++++ internal/schema/resource_data.go | 13 ++ 7 files changed, 197 insertions(+), 96 deletions(-) diff --git a/internal/providers/terraform/aws/testdata/dynamodb_table_china_test/dynamodb_table_china_test.golden b/internal/providers/terraform/aws/testdata/dynamodb_table_china_test/dynamodb_table_china_test.golden index c75fe9ac45a..ff5a3f39e42 100644 --- a/internal/providers/terraform/aws/testdata/dynamodb_table_china_test/dynamodb_table_china_test.golden +++ b/internal/providers/terraform/aws/testdata/dynamodb_table_china_test/dynamodb_table_china_test.golden @@ -1,16 +1,16 @@ - Name Monthly Qty Unit Monthly Cost (CNY) - - aws_dynamodb_table.my_dynamodb_table_china - ├─ Write capacity unit (WCU) 20 WCU 84.68 元 - ├─ Read capacity unit (RCU) 30 RCU 25.40 元 - ├─ Data storage Monthly cost depends on usage: 2.20 元 per GB - ├─ Point-In-Time Recovery (PITR) backup storage Monthly cost depends on usage: 1.76 元 per GB - ├─ On-demand backup storage Monthly cost depends on usage: 0.88 元 per GB - ├─ Table data restored Monthly cost depends on usage: 1.32 元 per GB - └─ Streams read request unit (sRRU) Monthly cost depends on usage: 0.00000178 元 per sRRUs - - OVERALL TOTAL (CNY) 110.08 元 + Name Monthly Qty Unit Monthly Cost (CNY) + + aws_dynamodb_table.my_dynamodb_table_china + ├─ Write capacity unit (WCU) 20 WCU 11.63 元 + ├─ Read capacity unit (RCU) 30 RCU 3.49 元 + ├─ Data storage Monthly cost depends on usage: 0.30 元 per GB + ├─ Point-In-Time Recovery (PITR) backup storage Monthly cost depends on usage: 0.24 元 per GB + ├─ On-demand backup storage Monthly cost depends on usage: 0.12 元 per GB + ├─ Table data restored Monthly cost depends on usage: 0.18 元 per GB + └─ Streams read request unit (sRRU) Monthly cost depends on usage: 0.0000002444 元 per sRRUs + + OVERALL TOTAL (CNY) 15.11 元 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -21,5 +21,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ 110 元 ┃ - ┃ 110 元 ┃ +┃ main ┃ 15 元 ┃ - ┃ 15 元 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/aws/testdata/lambda_function_china_test/lambda_function_china_test.golden b/internal/providers/terraform/aws/testdata/lambda_function_china_test/lambda_function_china_test.golden index 7d56a4b2226..d9c3c81731c 100644 --- a/internal/providers/terraform/aws/testdata/lambda_function_china_test/lambda_function_china_test.golden +++ b/internal/providers/terraform/aws/testdata/lambda_function_china_test/lambda_function_china_test.golden @@ -2,17 +2,17 @@ Name Monthly Qty Unit Monthly Cost (CNY) aws_lambda_function.lambda_china_with_usage - └─ Requests 0.1 1M requests 0.14 元 * + └─ Requests 0.1 1M requests 0.02 元 * aws_lambda_function.lambda_china_with_usage_arm - └─ Requests 0.1 1M requests 0.14 元 * + └─ Requests 0.1 1M requests 0.02 元 * aws_lambda_function.lambda_china - ├─ Requests Monthly cost depends on usage: 1.36 元 per 1M requests - ├─ Ephemeral storage Monthly cost depends on usage: 0.0000002303 元 per GB-seconds - └─ Duration Monthly cost depends on usage: 0.000113477 元 per GB-seconds + ├─ Requests Monthly cost depends on usage: 0.19 元 per 1M requests + ├─ Ephemeral storage Monthly cost depends on usage: 0.0000000316 元 per GB-seconds + └─ Duration Monthly cost depends on usage: 0.0000155804 元 per GB-seconds - OVERALL TOTAL (CNY) 0.27 元 + OVERALL TOTAL (CNY) 0.04 元 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -23,5 +23,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ 0.00 元 ┃ 0.27 元 ┃ 0.27 元 ┃ +┃ main ┃ 0.00 元 ┃ 0.04 元 ┃ 0.04 元 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/aws/testdata/rds_cluster_china_test/rds_cluster_china_test.golden b/internal/providers/terraform/aws/testdata/rds_cluster_china_test/rds_cluster_china_test.golden index 7288856ef37..2455793ae5a 100644 --- a/internal/providers/terraform/aws/testdata/rds_cluster_china_test/rds_cluster_china_test.golden +++ b/internal/providers/terraform/aws/testdata/rds_cluster_china_test/rds_cluster_china_test.golden @@ -2,8 +2,8 @@ Name Monthly Qty Unit Monthly Cost (CNY) aws_rds_cluster.mysql_china - ├─ Storage Monthly cost depends on usage: 0.67 元 per GB - ├─ I/O requests Monthly cost depends on usage: 1.33 元 per 1M requests + ├─ Storage Monthly cost depends on usage: 0.091991 元 per GB + ├─ I/O requests Monthly cost depends on usage: 0.18 元 per 1M requests ├─ Backup storage not found └─ Snapshot export not found diff --git a/internal/providers/terraform/aws/testdata/s3_bucket_china_test/s3_bucket_china_test.golden b/internal/providers/terraform/aws/testdata/s3_bucket_china_test/s3_bucket_china_test.golden index 81a58302236..f842cc129d9 100644 --- a/internal/providers/terraform/aws/testdata/s3_bucket_china_test/s3_bucket_china_test.golden +++ b/internal/providers/terraform/aws/testdata/s3_bucket_china_test/s3_bucket_china_test.golden @@ -1,77 +1,77 @@ - Name Monthly Qty Unit Monthly Cost (CNY) - - aws_s3_bucket.bucket_china_with_usage - ├─ Standard - │ ├─ Storage 10,000 GB 1,950.00 元 * - │ ├─ PUT, COPY, POST, LIST requests 10 1k requests 0.05 元 * - │ ├─ GET, SELECT, and all other requests 10 1k requests 0.02 元 * - │ ├─ Select data scanned 10,000 GB 159.30 元 * - │ └─ Select data returned 10,000 GB 57.00 元 * - ├─ Intelligent tiering - │ ├─ Storage (frequent access) 20,000 GB 3,900.00 元 * - │ ├─ Storage (infrequent access) 20,000 GB 2,675.40 元 * - │ ├─ Storage (archive access) 20,000 GB 601.20 元 * - │ ├─ Storage (deep archive access) 20,000 GB 267.20 元 * - │ ├─ Monitoring and automation 20 1k objects 0.33 元 * - │ ├─ PUT, COPY, POST, LIST requests 20 1k requests 0.09 元 * - │ ├─ GET, SELECT, and all other requests 20 1k requests 0.03 元 * - │ ├─ Lifecycle transition 20 1k requests 1.27 元 * - │ ├─ Select data scanned 20,000 GB 318.60 元 * - │ ├─ Select data returned 20,000 GB 114.00 元 * - │ └─ Early delete (within 30 days) 20,000 GB 3,900.00 元 * - ├─ Standard - infrequent access - │ ├─ Storage 30,000 GB 4,013.10 元 * - │ ├─ PUT, COPY, POST, LIST requests 30 1k requests 1.91 元 * - │ ├─ GET, SELECT, and all other requests 30 1k requests 0.19 元 * - │ ├─ Lifecycle transition 30 1k requests 1.91 元 * - │ ├─ Retrievals 30,000 GB 1,911.00 元 * - │ ├─ Select data scanned 30,000 GB 477.90 元 * - │ └─ Select data returned 30,000 GB 1,911.00 元 * - ├─ One zone - infrequent access - │ ├─ Storage 40,000 GB 4,280.64 元 * - │ ├─ PUT, COPY, POST, LIST requests 40 1k requests 2.55 元 * - │ ├─ GET, SELECT, and all other requests 40 1k requests 0.25 元 * - │ ├─ Lifecycle transition 40 1k requests 2.55 元 * - │ ├─ Retrievals 40,000 GB 2,548.00 元 * - │ ├─ Select data scanned 40,000 GB 637.20 元 * - │ └─ Select data returned 40,000 GB 2,548.00 元 * - ├─ Glacier flexible retrieval - │ ├─ Storage 50,000 GB 1,503.00 元 * - │ ├─ PUT, COPY, POST, LIST requests 50 1k requests 11.20 元 * - │ ├─ GET, SELECT, and all other requests 50 1k requests 0.08 元 * - │ ├─ Lifecycle transition 50 1k requests 11.20 元 * - │ ├─ Retrieval requests (standard) 50 1k requests 11.20 元 * - │ ├─ Retrievals (standard) 50,000 GB 3,335.00 元 * - │ ├─ Select data scanned (standard) 50,000 GB 2,800.00 元 * - │ ├─ Select data returned (standard) 50,000 GB 3,335.00 元 * - │ ├─ Retrieval requests (expedited) 50 1k requests 3,335.00 元 * - │ ├─ Retrievals (expedited) 50,000 GB 10,005.00 元 * - │ ├─ Select data scanned (expedited) 50,000 GB 7,005.00 元 * - │ ├─ Select data returned (expedited) 50,000 GB 10,005.00 元 * - │ ├─ Select data scanned (bulk) 50,000 GB 335.00 元 * - │ ├─ Select data returned (bulk) 50,000 GB 835.00 元 * - │ └─ Early delete (within 90 days) 50,000 GB 1,503.00 元 * - └─ Glacier deep archive - ├─ Storage 60,000 GB 801.60 元 * - ├─ PUT, COPY, POST, LIST requests 60 1k requests 26.87 元 * - ├─ GET, SELECT, and all other requests 60 1k requests 0.09 元 * - ├─ Lifecycle transition 60 1k requests 26.87 元 * - ├─ Retrieval requests (standard) 60 1k requests 44.78 元 * - ├─ Retrievals (standard) 60,000 GB 8,004.00 元 * - ├─ Retrieval requests (bulk) 60 1k requests 10.01 元 * - ├─ Retrievals (bulk) 60,000 GB 2,004.00 元 * - └─ Early delete (within 180 days) 60,000 GB 801.60 元 * - - aws_s3_bucket.bucket_china - └─ Standard - ├─ Storage Monthly cost depends on usage: 0.20 元 per GB - ├─ PUT, COPY, POST, LIST requests Monthly cost depends on usage: 0.0045 元 per 1k requests - ├─ GET, SELECT, and all other requests Monthly cost depends on usage: 0.0015 元 per 1k requests - ├─ Select data scanned Monthly cost depends on usage: 0.01593 元 per GB - └─ Select data returned Monthly cost depends on usage: 0.0057 元 per GB - - OVERALL TOTAL (CNY) 88,030.17 元 + Name Monthly Qty Unit Monthly Cost (CNY) + + aws_s3_bucket.bucket_china_with_usage + ├─ Standard + │ ├─ Storage 10,000 GB 267.74 元 * + │ ├─ PUT, COPY, POST, LIST requests 10 1k requests 0.01 元 * + │ ├─ GET, SELECT, and all other requests 10 1k requests 0.00 元 * + │ ├─ Select data scanned 10,000 GB 21.87 元 * + │ └─ Select data returned 10,000 GB 7.83 元 * + ├─ Intelligent tiering + │ ├─ Storage (frequent access) 20,000 GB 535.47 元 * + │ ├─ Storage (infrequent access) 20,000 GB 367.33 元 * + │ ├─ Storage (archive access) 20,000 GB 82.54 元 * + │ ├─ Storage (deep archive access) 20,000 GB 36.69 元 * + │ ├─ Monitoring and automation 20 1k objects 0.05 元 * + │ ├─ PUT, COPY, POST, LIST requests 20 1k requests 0.01 元 * + │ ├─ GET, SELECT, and all other requests 20 1k requests 0.00 元 * + │ ├─ Lifecycle transition 20 1k requests 0.17 元 * + │ ├─ Select data scanned 20,000 GB 43.74 元 * + │ ├─ Select data returned 20,000 GB 15.65 元 * + │ └─ Early delete (within 30 days) 20,000 GB 535.47 元 * + ├─ Standard - infrequent access + │ ├─ Storage 30,000 GB 551.00 元 * + │ ├─ PUT, COPY, POST, LIST requests 30 1k requests 0.26 元 * + │ ├─ GET, SELECT, and all other requests 30 1k requests 0.03 元 * + │ ├─ Lifecycle transition 30 1k requests 0.26 元 * + │ ├─ Retrievals 30,000 GB 262.38 元 * + │ ├─ Select data scanned 30,000 GB 65.62 元 * + │ └─ Select data returned 30,000 GB 262.38 元 * + ├─ One zone - infrequent access + │ ├─ Storage 40,000 GB 587.73 元 * + │ ├─ PUT, COPY, POST, LIST requests 40 1k requests 0.35 元 * + │ ├─ GET, SELECT, and all other requests 40 1k requests 0.03 元 * + │ ├─ Lifecycle transition 40 1k requests 0.35 元 * + │ ├─ Retrievals 40,000 GB 349.84 元 * + │ ├─ Select data scanned 40,000 GB 87.49 元 * + │ └─ Select data returned 40,000 GB 349.84 元 * + ├─ Glacier flexible retrieval + │ ├─ Storage 50,000 GB 206.36 元 * + │ ├─ PUT, COPY, POST, LIST requests 50 1k requests 1.54 元 * + │ ├─ GET, SELECT, and all other requests 50 1k requests 0.01 元 * + │ ├─ Lifecycle transition 50 1k requests 1.54 元 * + │ ├─ Retrieval requests (standard) 50 1k requests 1.54 元 * + │ ├─ Retrievals (standard) 50,000 GB 457.90 元 * + │ ├─ Select data scanned (standard) 50,000 GB 384.44 元 * + │ ├─ Select data returned (standard) 50,000 GB 457.90 元 * + │ ├─ Retrieval requests (expedited) 50 1k requests 457.90 元 * + │ ├─ Retrievals (expedited) 50,000 GB 1,373.69 元 * + │ ├─ Select data scanned (expedited) 50,000 GB 961.79 元 * + │ ├─ Select data returned (expedited) 50,000 GB 1,373.69 元 * + │ ├─ Select data scanned (bulk) 50,000 GB 46.00 元 * + │ ├─ Select data returned (bulk) 50,000 GB 114.65 元 * + │ └─ Early delete (within 90 days) 50,000 GB 206.36 元 * + └─ Glacier deep archive + ├─ Storage 60,000 GB 110.06 元 * + ├─ PUT, COPY, POST, LIST requests 60 1k requests 3.69 元 * + ├─ GET, SELECT, and all other requests 60 1k requests 0.01 元 * + ├─ Lifecycle transition 60 1k requests 3.69 元 * + ├─ Retrieval requests (standard) 60 1k requests 6.15 元 * + ├─ Retrievals (standard) 60,000 GB 1,098.95 元 * + ├─ Retrieval requests (bulk) 60 1k requests 1.37 元 * + ├─ Retrievals (bulk) 60,000 GB 275.15 元 * + └─ Early delete (within 180 days) 60,000 GB 110.06 元 * + + aws_s3_bucket.bucket_china + └─ Standard + ├─ Storage Monthly cost depends on usage: 0.0267735 元 per GB + ├─ PUT, COPY, POST, LIST requests Monthly cost depends on usage: 0.0006179 元 per 1k requests + ├─ GET, SELECT, and all other requests Monthly cost depends on usage: 0.000206 元 per 1k requests + ├─ Select data scanned Monthly cost depends on usage: 0.002187189 元 per GB + └─ Select data returned Monthly cost depends on usage: 0.00078261 元 per GB + + OVERALL TOTAL (CNY) 12,086.54 元 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -82,5 +82,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ 0.00 元 ┃ 88,030 元 ┃ 88,030 元 ┃ +┃ main ┃ 0.00 元 ┃ 12,087 元 ┃ 12,087 元 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/parser.go b/internal/providers/terraform/parser.go index 5a6e687a31e..503e9e8aadb 100644 --- a/internal/providers/terraform/parser.go +++ b/internal/providers/terraform/parser.go @@ -759,6 +759,8 @@ func (p *Parser) parseReferences(resData map[string]*schema.ResourceData, confLo } } } + + fixKnownModuleRefIssues(resData) } func (p *Parser) parseConfReferences(resData map[string]*schema.ResourceData, confLoader *ConfLoader, d *schema.ResourceData, attr string, registryMap *RegistryItemMap) bool { @@ -1184,6 +1186,48 @@ func parseKnownModuleRefs(resData map[string]*schema.ResourceData, confLoader *C } } +// fixKnownModuleRefIssues deals with edge cases where the module returns an id that doesn't work for the resource +// this is exemplified by the s3-bucket module which coalesces a number of id's starting with "aws_s3_bucket_policy." +// this means we're referencing the wrong resource in the plan JSON. This function fixes that by replacing the reference. +// the intention is to be laser focused in the application of this where we know the specific conditions it will occur +func fixKnownModuleRefIssues(resData map[string]*schema.ResourceData) { + knownRefs := []struct { + SourceResourceType string + AttributeName string + TargetResource string + ReplacementResource string + }{ + { + SourceResourceType: "aws_s3_bucket_lifecycle_configuration", + AttributeName: "bucket", + TargetResource: "aws_s3_bucket_policy", + ReplacementResource: "aws_s3_bucket", + }, + } + + for _, d := range resData { + for _, knownRef := range knownRefs { + if d.Type == knownRef.SourceResourceType { + for _, ref := range d.ReferencesMap[knownRef.AttributeName] { + if ref.Type == knownRef.TargetResource { + targetAddress := strings.Replace(ref.Address, knownRef.TargetResource, knownRef.ReplacementResource, 1) + + for _, target := range resData { + // find possible targets and ensure that its from the same module as existing reference + if target.Type == knownRef.ReplacementResource && target.Address == targetAddress { + // replace the reference + d.ReplaceReference(knownRef.AttributeName, ref, target) + + } + } + } + } + } + } + + } +} + func gjsonEqual(a, b gjson.Result) bool { var err error diff --git a/internal/providers/terraform/parser_test.go b/internal/providers/terraform/parser_test.go index aeb62bc2d5e..6a851164e7d 100644 --- a/internal/providers/terraform/parser_test.go +++ b/internal/providers/terraform/parser_test.go @@ -1,6 +1,7 @@ package terraform import ( + "github.com/stretchr/testify/require" "testing" "github.com/shopspring/decimal" @@ -761,6 +762,49 @@ func TestParseReferences_state(t *testing.T) { assert.Equal(t, []*schema.ResourceData{vol1}, resData["aws_ebs_snapshot.snapshot1"].References("volume_id")) } +func TestFixKnownModuleRefIssues(t *testing.T) { + bucket := schema.NewResourceData("aws_s3_bucket", "registry.terraform.io/hashicorp/aws", "module.bucket.aws_s3_bucket.this[0]", nil, gjson.Result{ + Type: gjson.JSON, + Raw: `{ + "id": "hcl-bucket-id" + "acl":"private", + "bucket":"my-bucket" + }`, + }) + + policy := schema.NewResourceData("aws_s3_bucket_policy", "registry.terraform.io/hashicorp/aws", "module.bucket.aws_s3_bucket_policy.this[0]", nil, gjson.Result{ + Type: gjson.JSON, + Raw: `{ + "id": "hcl-policy-id", + "bucket":"my-bucket", + "policy":"{}" + }`, + }) + + lifecycleConfiguration := schema.NewResourceData("aws_s3_bucket_lifecycle_configuration", "registry.terraform.io/hashicorp/aws", "aws_s3_bucket_lifecycle_configuration.bucket_lifecycle", nil, gjson.Result{ + Type: gjson.JSON, + Raw: `{ + "bucket": "hcl-policy-id" + }`, + }) + + lifecycleConfiguration.AddReference("bucket", policy, []string{}) + + resData := map[string]*schema.ResourceData{ + lifecycleConfiguration.Address: lifecycleConfiguration, + policy.Address: policy, + bucket.Address: bucket, + } + + require.Equal(t, lifecycleConfiguration.Get("bucket").String(), "hcl-policy-id") + assert.Equal(t, lifecycleConfiguration.References("bucket")[0].Type, "aws_s3_bucket_policy") + fixKnownModuleRefIssues(resData) + + assert.Equal(t, lifecycleConfiguration.Get("bucket").String(), "hcl-bucket-id") + assert.Equal(t, lifecycleConfiguration.References("bucket")[0].Get("id").String(), "hcl-bucket-id") + assert.Equal(t, lifecycleConfiguration.References("bucket")[0].Type, "aws_s3_bucket") +} + func TestParseKnownModuleRefs(t *testing.T) { res := schema.NewResourceData("aws_autoscaling_group", "registry.terraform.io/hashicorp/aws", "module.worker_groups_launch_template.aws_autoscaling_group.workers_launch_template[0]", nil, gjson.Result{ Type: gjson.JSON, diff --git a/internal/schema/resource_data.go b/internal/schema/resource_data.go index 40ced66e82e..10baa5c4941 100644 --- a/internal/schema/resource_data.go +++ b/internal/schema/resource_data.go @@ -142,6 +142,19 @@ func (d *ResourceData) AddReference(k string, reference *ResourceData, reverseRe } } +func (d *ResourceData) ReplaceReference(k string, oldReference *ResourceData, newReference *ResourceData) { + key := strings.Clone(k) + for i, r := range d.ReferencesMap[key] { + if r == oldReference { + d.ReferencesMap[key][i] = newReference + break + } + } + + // update the raw values on the resource data + d.Set(key, newReference.Get("id").String()) +} + func (d *ResourceData) Set(key string, value interface{}) { d.RawValues = AddRawValue(d.RawValues, key, value) } From fa442a9a1fa324dce1d45b624b42ebfab9d55958 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Tue, 17 Dec 2024 14:28:32 +0000 Subject: [PATCH 61/81] chore(deps): switch to maintained package of git-urls --- go.mod | 2 +- go.sum | 4 ++-- internal/hcl/modules/loader.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index a0085504e6d..456cd97179f 100644 --- a/go.mod +++ b/go.mod @@ -93,6 +93,7 @@ require ( require ( github.com/alecthomas/jsonschema v0.0.0-20211209230136-e2b41affa5c1 + github.com/chainguard-dev/git-urls v1.0.2 github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 github.com/fatih/camelcase v1.0.0 github.com/go-errors/errors v1.4.2 @@ -117,7 +118,6 @@ require ( github.com/soongo/path-to-regexp v1.6.4 github.com/spacelift-io/spacectl v1.2.0 github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7 - github.com/whilp/git-urls v1.0.0 github.com/withfig/autocomplete-tools/packages/cobra v1.2.0 github.com/xanzy/go-gitlab v0.86.0 golang.org/x/exp v0.0.0-20231006140011-7918f672742d diff --git a/go.sum b/go.sum index cb072ab5b72..1b8b541a72b 100644 --- a/go.sum +++ b/go.sum @@ -409,6 +409,8 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chainguard-dev/git-urls v1.0.2 h1:pSpT7ifrpc5X55n4aTTm7FFUE+ZQHKiqpiwNkJrVcKQ= +github.com/chainguard-dev/git-urls v1.0.2/go.mod h1:rbGgj10OS7UgZlbzdUQIQpT0k/D4+An04HJY7Ol+Y/o= github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:o64h9XF42kVEUuhuer2ehqrlX8rZmvQSU0+Vpj1rF6Q= github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:Rp8e0DCtEKwXFOC6JPJQVTz8tuGoGvw6Xfexggh/ed0= github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= @@ -1163,8 +1165,6 @@ github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= -github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU= -github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE= github.com/withfig/autocomplete-tools/packages/cobra v1.2.0 h1:MzD3XeOOSO3mAjOPpF07jFteSKZxsRHvlIcAR9RQzKM= github.com/withfig/autocomplete-tools/packages/cobra v1.2.0/go.mod h1:RoXh7+7qknOXL65uTzdzE1mPxqcPwS7FLCE9K5GfmKo= github.com/xanzy/go-gitlab v0.86.0 h1:jR8V9cK9jXRQDb46KOB20NCF3ksY09luaG0IfXE6p7w= diff --git a/internal/hcl/modules/loader.go b/internal/hcl/modules/loader.go index c726d3db902..d407756b064 100644 --- a/internal/hcl/modules/loader.go +++ b/internal/hcl/modules/loader.go @@ -16,13 +16,13 @@ import ( "sync" "time" + giturls "github.com/chainguard-dev/git-urls" "github.com/hashicorp/go-getter" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/terraform-config-inspect/tfconfig" "github.com/infracost/infracost/internal/metrics" "github.com/otiai10/copy" "github.com/rs/zerolog" - giturls "github.com/whilp/git-urls" "golang.org/x/sync/errgroup" "github.com/infracost/infracost/internal/config" From 24677e642bc3823f5ec8d267fb1169ac0b564cff Mon Sep 17 00:00:00 2001 From: Hugo Rut Date: Wed, 18 Dec 2024 14:00:47 +0100 Subject: [PATCH 62/81] feat: add detected env name to input vars/workspace (#3278) adds the inferred env name as a starting variable input to the terraform project if no user workspace has been defined. This is preferable over using just the base "default" workspace. --- cmd/infracost/breakdown_test.go | 5 ++ ...own_terraform_provided_default_envs.golden | 49 +++++++++++++++++++ .../infracost.yml | 8 +++ .../mod/main.tf | 33 +++++++++++++ internal/hcl/parser.go | 26 ++++++++-- 5 files changed, 118 insertions(+), 3 deletions(-) create mode 100644 cmd/infracost/testdata/breakdown_terraform_provided_default_envs/breakdown_terraform_provided_default_envs.golden create mode 100644 cmd/infracost/testdata/breakdown_terraform_provided_default_envs/infracost.yml create mode 100644 cmd/infracost/testdata/breakdown_terraform_provided_default_envs/mod/main.tf diff --git a/cmd/infracost/breakdown_test.go b/cmd/infracost/breakdown_test.go index f2648a647c7..c1675473bb1 100644 --- a/cmd/infracost/breakdown_test.go +++ b/cmd/infracost/breakdown_test.go @@ -356,6 +356,11 @@ func TestBreakdownTerraformDirectoryWithRecursiveModules(t *testing.T) { GoldenFileCommandTest(t, testutil.CalcGoldenFileTestdataDirName(), []string{"breakdown", "--path", dir}, &GoldenFileOptions{RunTerraformCLI: true}) } +func TestBreakdownTerraformProvidedDefaultEnvs(t *testing.T) { + dir := path.Join("./testdata", testutil.CalcGoldenFileTestdataDirName()) + GoldenFileCommandTest(t, testutil.CalcGoldenFileTestdataDirName(), []string{"breakdown", "--config-file", path.Join(dir, "infracost.yml")}, nil) +} + func TestBreakdownTerraformFieldsAll(t *testing.T) { GoldenFileCommandTest(t, testutil.CalcGoldenFileTestdataDirName(), []string{"breakdown", "--path", "./testdata/example_plan.json", "--usage-file", "./testdata/example_usage.yml", "--fields", "all"}, nil) } diff --git a/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/breakdown_terraform_provided_default_envs.golden b/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/breakdown_terraform_provided_default_envs.golden new file mode 100644 index 00000000000..757b4a85cf3 --- /dev/null +++ b/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/breakdown_terraform_provided_default_envs.golden @@ -0,0 +1,49 @@ +Project: dev +Module path: REPLACED_PROJECT_PATH/testdata/breakdown_terraform_provided_default_envs/mod + + Name Monthly Qty Unit Monthly Cost + + aws_instance.web_app + ├─ Instance usage (Linux/UNIX, on-demand, t2.micro) 730 hours $8.47 + ├─ root_block_device + │ └─ Storage (general purpose SSD, gp2) 50 GB $5.00 + └─ ebs_block_device[0] + ├─ Storage (provisioned IOPS SSD, io1) 1,000 GB $125.00 + └─ Provisioned IOPS 800 IOPS $52.00 + + Project total $190.47 + +────────────────────────────────── +Project: prod +Module path: REPLACED_PROJECT_PATH/testdata/breakdown_terraform_provided_default_envs/mod +Workspace: stg + + Name Monthly Qty Unit Monthly Cost + + aws_instance.web_app + ├─ Instance usage (Linux/UNIX, on-demand, t3.large) 730 hours $60.74 + ├─ root_block_device + │ └─ Storage (general purpose SSD, gp2) 50 GB $5.00 + └─ ebs_block_device[0] + ├─ Storage (provisioned IOPS SSD, io1) 1,000 GB $125.00 + └─ Provisioned IOPS 800 IOPS $52.00 + + Project total $242.74 + + OVERALL TOTAL $433.20 + +*Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. + +────────────────────────────────── +2 cloud resources were detected: +∙ 2 were estimated + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ +┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ +┃ dev ┃ $190 ┃ - ┃ $190 ┃ +┃ prod ┃ $243 ┃ - ┃ $243 ┃ +┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ + +Err: + diff --git a/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/infracost.yml b/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/infracost.yml new file mode 100644 index 00000000000..afd35cff37d --- /dev/null +++ b/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/infracost.yml @@ -0,0 +1,8 @@ +version: 0.1 + +projects: + - path: "./testdata/breakdown_terraform_provided_default_envs/mod" + name: "dev" + - path: "./testdata/breakdown_terraform_provided_default_envs/mod" + name: "prod" + terraform_workspace: "stg" diff --git a/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/mod/main.tf b/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/mod/main.tf new file mode 100644 index 00000000000..245204f2241 --- /dev/null +++ b/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/mod/main.tf @@ -0,0 +1,33 @@ +provider "aws" { + region = "us-east-1" + skip_credentials_validation = true + skip_requesting_account_id = true + access_key = "mock_access_key" + secret_key = "mock_secret_key" +} + +variable "env" {} + +locals { + instance_types = { + dev = "t2.micro" + prod = "t3.medium" + stg = "t3.large" + } +} + +resource "aws_instance" "web_app" { + ami = "ami-674cbc1e" + instance_type = local.instance_types[var.env] + + root_block_device { + volume_size = 50 + } + + ebs_block_device { + device_name = "my_data" + volume_type = "io1" + volume_size = 1000 + iops = 800 + } +} diff --git a/internal/hcl/parser.go b/internal/hcl/parser.go index bfa4421bfaa..68286e8bafc 100644 --- a/internal/hcl/parser.go +++ b/internal/hcl/parser.go @@ -11,12 +11,13 @@ import ( "strings" "github.com/hashicorp/hcl/v2" - "github.com/infracost/infracost/internal/metrics" "github.com/rs/zerolog" "github.com/zclconf/go-cty/cty" "github.com/zclconf/go-cty/cty/gocty" "golang.org/x/exp/maps" + "github.com/infracost/infracost/internal/metrics" + "github.com/infracost/infracost/internal/clierror" "github.com/infracost/infracost/internal/config" "github.com/infracost/infracost/internal/extclient" @@ -334,7 +335,6 @@ func NewParser(projectRoot RootPath, envMatcher *EnvFileMatcher, moduleLoader *m detectedProjectPath: projectRoot.DetectedPath, hasChanges: projectRoot.HasChanges, moduleCalls: projectRoot.ModuleCalls, - workspaceName: defaultTerraformWorkspaceName, hclParser: hclParser, blockBuilder: BlockBuilder{SetAttributes: []SetAttributesFunc{SetUUIDAttributes}, Logger: logger, HCLParser: hclParser}, logger: parserLogger, @@ -346,6 +346,12 @@ func NewParser(projectRoot RootPath, envMatcher *EnvFileMatcher, moduleLoader *m option(p) } + // if the project name is not set by the user, we will use the detected env name + // as the project name. + if p.workspaceName == "" && p.ProjectName() != p.EnvName() { + p.workspaceName = p.EnvName() + } + return p } @@ -589,7 +595,7 @@ func (p *Parser) ProjectName() string { if p.moduleSuffix != "" { name = fmt.Sprintf("%s-%s", name, p.moduleSuffix) - } else if p.workspaceName != "" && p.workspaceName != defaultTerraformWorkspaceName { + } else if p.workspaceName != "" && p.workspaceName != defaultTerraformWorkspaceName && !strings.HasSuffix(name, p.workspaceName) { name = fmt.Sprintf("%s-%s", name, p.workspaceName) } @@ -686,6 +692,20 @@ func (p *Parser) loadVars(blocks Blocks, filenames []string) (map[string]cty.Val combinedVars[k] = v } + // add a common "env" name to the input vars for the project if it is not + // explicitly defined. This is done as often users have a common project variable + // called "env" that is used to define the environment name. Adding this default + // allows us to have a somewhat sane fallback for situations where we + // fail to provide the input from a var file or similar config. + if _, ok := combinedVars["env"]; !ok { + env := p.workspaceName + if env == "" { + env = p.EnvName() + } + + combinedVars["env"] = cty.StringVal(env) + } + return combinedVars, nil } From a9e0820a899cda1f830779192f75edd6bddef5c3 Mon Sep 17 00:00:00 2001 From: Hugo Rut Date: Wed, 18 Dec 2024 14:19:30 +0100 Subject: [PATCH 63/81] Revert "feat: add detected env name to input vars/workspace (#3278)" (#3279) This reverts commit 24677e642bc3823f5ec8d267fb1169ac0b564cff. --- cmd/infracost/breakdown_test.go | 5 -- ...own_terraform_provided_default_envs.golden | 49 ------------------- .../infracost.yml | 8 --- .../mod/main.tf | 33 ------------- internal/hcl/parser.go | 26 ++-------- 5 files changed, 3 insertions(+), 118 deletions(-) delete mode 100644 cmd/infracost/testdata/breakdown_terraform_provided_default_envs/breakdown_terraform_provided_default_envs.golden delete mode 100644 cmd/infracost/testdata/breakdown_terraform_provided_default_envs/infracost.yml delete mode 100644 cmd/infracost/testdata/breakdown_terraform_provided_default_envs/mod/main.tf diff --git a/cmd/infracost/breakdown_test.go b/cmd/infracost/breakdown_test.go index c1675473bb1..f2648a647c7 100644 --- a/cmd/infracost/breakdown_test.go +++ b/cmd/infracost/breakdown_test.go @@ -356,11 +356,6 @@ func TestBreakdownTerraformDirectoryWithRecursiveModules(t *testing.T) { GoldenFileCommandTest(t, testutil.CalcGoldenFileTestdataDirName(), []string{"breakdown", "--path", dir}, &GoldenFileOptions{RunTerraformCLI: true}) } -func TestBreakdownTerraformProvidedDefaultEnvs(t *testing.T) { - dir := path.Join("./testdata", testutil.CalcGoldenFileTestdataDirName()) - GoldenFileCommandTest(t, testutil.CalcGoldenFileTestdataDirName(), []string{"breakdown", "--config-file", path.Join(dir, "infracost.yml")}, nil) -} - func TestBreakdownTerraformFieldsAll(t *testing.T) { GoldenFileCommandTest(t, testutil.CalcGoldenFileTestdataDirName(), []string{"breakdown", "--path", "./testdata/example_plan.json", "--usage-file", "./testdata/example_usage.yml", "--fields", "all"}, nil) } diff --git a/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/breakdown_terraform_provided_default_envs.golden b/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/breakdown_terraform_provided_default_envs.golden deleted file mode 100644 index 757b4a85cf3..00000000000 --- a/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/breakdown_terraform_provided_default_envs.golden +++ /dev/null @@ -1,49 +0,0 @@ -Project: dev -Module path: REPLACED_PROJECT_PATH/testdata/breakdown_terraform_provided_default_envs/mod - - Name Monthly Qty Unit Monthly Cost - - aws_instance.web_app - ├─ Instance usage (Linux/UNIX, on-demand, t2.micro) 730 hours $8.47 - ├─ root_block_device - │ └─ Storage (general purpose SSD, gp2) 50 GB $5.00 - └─ ebs_block_device[0] - ├─ Storage (provisioned IOPS SSD, io1) 1,000 GB $125.00 - └─ Provisioned IOPS 800 IOPS $52.00 - - Project total $190.47 - -────────────────────────────────── -Project: prod -Module path: REPLACED_PROJECT_PATH/testdata/breakdown_terraform_provided_default_envs/mod -Workspace: stg - - Name Monthly Qty Unit Monthly Cost - - aws_instance.web_app - ├─ Instance usage (Linux/UNIX, on-demand, t3.large) 730 hours $60.74 - ├─ root_block_device - │ └─ Storage (general purpose SSD, gp2) 50 GB $5.00 - └─ ebs_block_device[0] - ├─ Storage (provisioned IOPS SSD, io1) 1,000 GB $125.00 - └─ Provisioned IOPS 800 IOPS $52.00 - - Project total $242.74 - - OVERALL TOTAL $433.20 - -*Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. - -────────────────────────────────── -2 cloud resources were detected: -∙ 2 were estimated - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ -┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ dev ┃ $190 ┃ - ┃ $190 ┃ -┃ prod ┃ $243 ┃ - ┃ $243 ┃ -┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ - -Err: - diff --git a/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/infracost.yml b/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/infracost.yml deleted file mode 100644 index afd35cff37d..00000000000 --- a/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/infracost.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: 0.1 - -projects: - - path: "./testdata/breakdown_terraform_provided_default_envs/mod" - name: "dev" - - path: "./testdata/breakdown_terraform_provided_default_envs/mod" - name: "prod" - terraform_workspace: "stg" diff --git a/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/mod/main.tf b/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/mod/main.tf deleted file mode 100644 index 245204f2241..00000000000 --- a/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/mod/main.tf +++ /dev/null @@ -1,33 +0,0 @@ -provider "aws" { - region = "us-east-1" - skip_credentials_validation = true - skip_requesting_account_id = true - access_key = "mock_access_key" - secret_key = "mock_secret_key" -} - -variable "env" {} - -locals { - instance_types = { - dev = "t2.micro" - prod = "t3.medium" - stg = "t3.large" - } -} - -resource "aws_instance" "web_app" { - ami = "ami-674cbc1e" - instance_type = local.instance_types[var.env] - - root_block_device { - volume_size = 50 - } - - ebs_block_device { - device_name = "my_data" - volume_type = "io1" - volume_size = 1000 - iops = 800 - } -} diff --git a/internal/hcl/parser.go b/internal/hcl/parser.go index 68286e8bafc..bfa4421bfaa 100644 --- a/internal/hcl/parser.go +++ b/internal/hcl/parser.go @@ -11,13 +11,12 @@ import ( "strings" "github.com/hashicorp/hcl/v2" + "github.com/infracost/infracost/internal/metrics" "github.com/rs/zerolog" "github.com/zclconf/go-cty/cty" "github.com/zclconf/go-cty/cty/gocty" "golang.org/x/exp/maps" - "github.com/infracost/infracost/internal/metrics" - "github.com/infracost/infracost/internal/clierror" "github.com/infracost/infracost/internal/config" "github.com/infracost/infracost/internal/extclient" @@ -335,6 +334,7 @@ func NewParser(projectRoot RootPath, envMatcher *EnvFileMatcher, moduleLoader *m detectedProjectPath: projectRoot.DetectedPath, hasChanges: projectRoot.HasChanges, moduleCalls: projectRoot.ModuleCalls, + workspaceName: defaultTerraformWorkspaceName, hclParser: hclParser, blockBuilder: BlockBuilder{SetAttributes: []SetAttributesFunc{SetUUIDAttributes}, Logger: logger, HCLParser: hclParser}, logger: parserLogger, @@ -346,12 +346,6 @@ func NewParser(projectRoot RootPath, envMatcher *EnvFileMatcher, moduleLoader *m option(p) } - // if the project name is not set by the user, we will use the detected env name - // as the project name. - if p.workspaceName == "" && p.ProjectName() != p.EnvName() { - p.workspaceName = p.EnvName() - } - return p } @@ -595,7 +589,7 @@ func (p *Parser) ProjectName() string { if p.moduleSuffix != "" { name = fmt.Sprintf("%s-%s", name, p.moduleSuffix) - } else if p.workspaceName != "" && p.workspaceName != defaultTerraformWorkspaceName && !strings.HasSuffix(name, p.workspaceName) { + } else if p.workspaceName != "" && p.workspaceName != defaultTerraformWorkspaceName { name = fmt.Sprintf("%s-%s", name, p.workspaceName) } @@ -692,20 +686,6 @@ func (p *Parser) loadVars(blocks Blocks, filenames []string) (map[string]cty.Val combinedVars[k] = v } - // add a common "env" name to the input vars for the project if it is not - // explicitly defined. This is done as often users have a common project variable - // called "env" that is used to define the environment name. Adding this default - // allows us to have a somewhat sane fallback for situations where we - // fail to provide the input from a var file or similar config. - if _, ok := combinedVars["env"]; !ok { - env := p.workspaceName - if env == "" { - env = p.EnvName() - } - - combinedVars["env"] = cty.StringVal(env) - } - return combinedVars, nil } From 73d15fd1e099b661aa9bb4c4401faa875d25a7d1 Mon Sep 17 00:00:00 2001 From: Hugo Rut Date: Wed, 18 Dec 2024 15:09:43 +0100 Subject: [PATCH 64/81] =?UTF-8?q?Revert=20"Revert=20"feat:=20add=20detecte?= =?UTF-8?q?d=20env=20name=20to=20input=20vars/workspace=20(#3278)"=20?= =?UTF-8?q?=E2=80=A6"=20(#3280)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit a9e0820a899cda1f830779192f75edd6bddef5c3. --- cmd/infracost/breakdown_test.go | 5 ++ ...own_terraform_provided_default_envs.golden | 49 +++++++++++++++++++ .../infracost.yml | 8 +++ .../mod/main.tf | 33 +++++++++++++ internal/hcl/parser.go | 26 ++++++++-- 5 files changed, 118 insertions(+), 3 deletions(-) create mode 100644 cmd/infracost/testdata/breakdown_terraform_provided_default_envs/breakdown_terraform_provided_default_envs.golden create mode 100644 cmd/infracost/testdata/breakdown_terraform_provided_default_envs/infracost.yml create mode 100644 cmd/infracost/testdata/breakdown_terraform_provided_default_envs/mod/main.tf diff --git a/cmd/infracost/breakdown_test.go b/cmd/infracost/breakdown_test.go index f2648a647c7..c1675473bb1 100644 --- a/cmd/infracost/breakdown_test.go +++ b/cmd/infracost/breakdown_test.go @@ -356,6 +356,11 @@ func TestBreakdownTerraformDirectoryWithRecursiveModules(t *testing.T) { GoldenFileCommandTest(t, testutil.CalcGoldenFileTestdataDirName(), []string{"breakdown", "--path", dir}, &GoldenFileOptions{RunTerraformCLI: true}) } +func TestBreakdownTerraformProvidedDefaultEnvs(t *testing.T) { + dir := path.Join("./testdata", testutil.CalcGoldenFileTestdataDirName()) + GoldenFileCommandTest(t, testutil.CalcGoldenFileTestdataDirName(), []string{"breakdown", "--config-file", path.Join(dir, "infracost.yml")}, nil) +} + func TestBreakdownTerraformFieldsAll(t *testing.T) { GoldenFileCommandTest(t, testutil.CalcGoldenFileTestdataDirName(), []string{"breakdown", "--path", "./testdata/example_plan.json", "--usage-file", "./testdata/example_usage.yml", "--fields", "all"}, nil) } diff --git a/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/breakdown_terraform_provided_default_envs.golden b/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/breakdown_terraform_provided_default_envs.golden new file mode 100644 index 00000000000..757b4a85cf3 --- /dev/null +++ b/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/breakdown_terraform_provided_default_envs.golden @@ -0,0 +1,49 @@ +Project: dev +Module path: REPLACED_PROJECT_PATH/testdata/breakdown_terraform_provided_default_envs/mod + + Name Monthly Qty Unit Monthly Cost + + aws_instance.web_app + ├─ Instance usage (Linux/UNIX, on-demand, t2.micro) 730 hours $8.47 + ├─ root_block_device + │ └─ Storage (general purpose SSD, gp2) 50 GB $5.00 + └─ ebs_block_device[0] + ├─ Storage (provisioned IOPS SSD, io1) 1,000 GB $125.00 + └─ Provisioned IOPS 800 IOPS $52.00 + + Project total $190.47 + +────────────────────────────────── +Project: prod +Module path: REPLACED_PROJECT_PATH/testdata/breakdown_terraform_provided_default_envs/mod +Workspace: stg + + Name Monthly Qty Unit Monthly Cost + + aws_instance.web_app + ├─ Instance usage (Linux/UNIX, on-demand, t3.large) 730 hours $60.74 + ├─ root_block_device + │ └─ Storage (general purpose SSD, gp2) 50 GB $5.00 + └─ ebs_block_device[0] + ├─ Storage (provisioned IOPS SSD, io1) 1,000 GB $125.00 + └─ Provisioned IOPS 800 IOPS $52.00 + + Project total $242.74 + + OVERALL TOTAL $433.20 + +*Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. + +────────────────────────────────── +2 cloud resources were detected: +∙ 2 were estimated + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ +┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ +┃ dev ┃ $190 ┃ - ┃ $190 ┃ +┃ prod ┃ $243 ┃ - ┃ $243 ┃ +┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ + +Err: + diff --git a/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/infracost.yml b/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/infracost.yml new file mode 100644 index 00000000000..afd35cff37d --- /dev/null +++ b/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/infracost.yml @@ -0,0 +1,8 @@ +version: 0.1 + +projects: + - path: "./testdata/breakdown_terraform_provided_default_envs/mod" + name: "dev" + - path: "./testdata/breakdown_terraform_provided_default_envs/mod" + name: "prod" + terraform_workspace: "stg" diff --git a/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/mod/main.tf b/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/mod/main.tf new file mode 100644 index 00000000000..245204f2241 --- /dev/null +++ b/cmd/infracost/testdata/breakdown_terraform_provided_default_envs/mod/main.tf @@ -0,0 +1,33 @@ +provider "aws" { + region = "us-east-1" + skip_credentials_validation = true + skip_requesting_account_id = true + access_key = "mock_access_key" + secret_key = "mock_secret_key" +} + +variable "env" {} + +locals { + instance_types = { + dev = "t2.micro" + prod = "t3.medium" + stg = "t3.large" + } +} + +resource "aws_instance" "web_app" { + ami = "ami-674cbc1e" + instance_type = local.instance_types[var.env] + + root_block_device { + volume_size = 50 + } + + ebs_block_device { + device_name = "my_data" + volume_type = "io1" + volume_size = 1000 + iops = 800 + } +} diff --git a/internal/hcl/parser.go b/internal/hcl/parser.go index bfa4421bfaa..68286e8bafc 100644 --- a/internal/hcl/parser.go +++ b/internal/hcl/parser.go @@ -11,12 +11,13 @@ import ( "strings" "github.com/hashicorp/hcl/v2" - "github.com/infracost/infracost/internal/metrics" "github.com/rs/zerolog" "github.com/zclconf/go-cty/cty" "github.com/zclconf/go-cty/cty/gocty" "golang.org/x/exp/maps" + "github.com/infracost/infracost/internal/metrics" + "github.com/infracost/infracost/internal/clierror" "github.com/infracost/infracost/internal/config" "github.com/infracost/infracost/internal/extclient" @@ -334,7 +335,6 @@ func NewParser(projectRoot RootPath, envMatcher *EnvFileMatcher, moduleLoader *m detectedProjectPath: projectRoot.DetectedPath, hasChanges: projectRoot.HasChanges, moduleCalls: projectRoot.ModuleCalls, - workspaceName: defaultTerraformWorkspaceName, hclParser: hclParser, blockBuilder: BlockBuilder{SetAttributes: []SetAttributesFunc{SetUUIDAttributes}, Logger: logger, HCLParser: hclParser}, logger: parserLogger, @@ -346,6 +346,12 @@ func NewParser(projectRoot RootPath, envMatcher *EnvFileMatcher, moduleLoader *m option(p) } + // if the project name is not set by the user, we will use the detected env name + // as the project name. + if p.workspaceName == "" && p.ProjectName() != p.EnvName() { + p.workspaceName = p.EnvName() + } + return p } @@ -589,7 +595,7 @@ func (p *Parser) ProjectName() string { if p.moduleSuffix != "" { name = fmt.Sprintf("%s-%s", name, p.moduleSuffix) - } else if p.workspaceName != "" && p.workspaceName != defaultTerraformWorkspaceName { + } else if p.workspaceName != "" && p.workspaceName != defaultTerraformWorkspaceName && !strings.HasSuffix(name, p.workspaceName) { name = fmt.Sprintf("%s-%s", name, p.workspaceName) } @@ -686,6 +692,20 @@ func (p *Parser) loadVars(blocks Blocks, filenames []string) (map[string]cty.Val combinedVars[k] = v } + // add a common "env" name to the input vars for the project if it is not + // explicitly defined. This is done as often users have a common project variable + // called "env" that is used to define the environment name. Adding this default + // allows us to have a somewhat sane fallback for situations where we + // fail to provide the input from a var file or similar config. + if _, ok := combinedVars["env"]; !ok { + env := p.workspaceName + if env == "" { + env = p.EnvName() + } + + combinedVars["env"] = cty.StringVal(env) + } + return combinedVars, nil } From d7df17f0c71ef3ae11436d4f36fbf36e59a6f671 Mon Sep 17 00:00:00 2001 From: Liam Galvin Date: Wed, 18 Dec 2024 14:32:55 +0000 Subject: [PATCH 65/81] fix: Workaround go-getter ctx issue and set timeout on git-getter directly (#3281) * fix: Workaround go-getter ctx issue and set timeout on git-getter directly * handle an override being used * simplify --- internal/hcl/modules/fetch.go | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/internal/hcl/modules/fetch.go b/internal/hcl/modules/fetch.go index 7d9ee4c32da..d276a4625fa 100644 --- a/internal/hcl/modules/fetch.go +++ b/internal/hcl/modules/fetch.go @@ -1,7 +1,6 @@ package modules import ( - "context" "fmt" "net/url" "os" @@ -33,7 +32,6 @@ type PackageFetcher struct { remoteCache RemoteCache logger zerolog.Logger getters map[string]getter.Getter - timeout time.Duration } // use a global cache to avoid downloading the same module multiple times for each project @@ -53,13 +51,16 @@ func NewPackageFetcher(remoteCache RemoteCache, logger zerolog.Logger, opts ...P for k, g := range getter.Getters { getters[k] = g } - getters["git"] = &CustomGitGetter{new(getter.GitGetter)} + getters["git"] = &CustomGitGetter{ + &getter.GitGetter{ + Timeout: defaultModuleRetrieveTimeout, + }, + } p := &PackageFetcher{ remoteCache: remoteCache, logger: logger, getters: getters, - timeout: defaultModuleRetrieveTimeout, } for _, opt := range opts { @@ -77,12 +78,6 @@ func WithGetters(getters map[string]getter.Getter) PackageFetcherOpts { } } -var WithTimeout = func(d time.Duration) PackageFetcherOpts { - return func(p *PackageFetcher) { - p.timeout = d - } -} - // fetch downloads the remote module using the go-getter library // See: https://github.com/hashicorp/go-getter func (p *PackageFetcher) Fetch(moduleAddr string, dest string) error { @@ -209,15 +204,7 @@ func (p *PackageFetcher) fetchFromRemote(moduleAddr, dest string) (bool, error) // I'm not sure if we really need it, but added it just in case/ decompressors["tar.tbz2"] = new(getter.TarBzip2Decompressor) - ctx := context.Background() - if p.timeout > 0 { - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, p.timeout) - defer cancel() - } - client := getter.Client{ - Ctx: ctx, Src: moduleAddr, Dst: dest, Pwd: dest, From 426be4502b3980434dc86d185fd9b85570a6e5b9 Mon Sep 17 00:00:00 2001 From: Hugo Rut Date: Thu, 19 Dec 2024 11:39:43 +0100 Subject: [PATCH 66/81] enhance: spacelift variable loader improvements (#3285) * fix: only use one spacelift stack per repo project Changes remote variable loading to ignore a matched stack if it has been previously used by another project. * fix: update spacelift matching logic to also match on prefix * fix: add tests --- internal/hcl/remote_variables_loader.go | 16 ++++- internal/hcl/remote_variables_loader_test.go | 66 ++++++++++++++++++++ 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/internal/hcl/remote_variables_loader.go b/internal/hcl/remote_variables_loader.go index 0a83ab468c8..5680c6b4839 100644 --- a/internal/hcl/remote_variables_loader.go +++ b/internal/hcl/remote_variables_loader.go @@ -379,7 +379,8 @@ type SpaceliftRemoteVariableLoader struct { Client client.Client Metadata vcs.Metadata - cache *sync.Map + cache *sync.Map + hitMap *sync.Map } // NewSpaceliftRemoteVariableLoader creates a new SpaceliftRemoteVariableLoader, this function @@ -396,6 +397,7 @@ func NewSpaceliftRemoteVariableLoader(metadata vcs.Metadata, apiKeyEndpoint, api Client: client.New(httpClient, session), Metadata: metadata, cache: &sync.Map{}, + hitMap: &sync.Map{}, }, nil } @@ -428,7 +430,7 @@ func (s *SpaceliftRemoteVariableLoader) Load(options RemoteVarLoaderOptions) (ma if options.Environment != "" { var filteredStacks []stack for _, stack := range stacks { - if strings.HasSuffix(stack.Name, fmt.Sprintf(":%s", options.Environment)) { + if strings.HasSuffix(stack.Name, fmt.Sprintf(":%s", options.Environment)) || strings.HasPrefix(stack.Name, fmt.Sprintf("%s/", options.Environment)) { filteredStacks = append(filteredStacks, stack) } } @@ -444,8 +446,16 @@ func (s *SpaceliftRemoteVariableLoader) Load(options RemoteVarLoaderOptions) (ma logging.Logger.Warn().Msgf("found multiple Spacelift stacks for module path %q with environment %q, skipping Spacelift remote variable loading", options.ModulePath, options.Environment) return nil, nil } + stack := stacks[0] + prevOptions, hit := s.hitMap.Load(stack.ID) + if hit { + logging.Logger.Debug().Msgf("spacelift stack %q already used for options %+v will not load remote variables", stack.ID, prevOptions) + return nil, nil + } + + s.hitMap.Store(stack.ID, options) - logging.Logger.Debug().Msgf("found Spacelift stack %q for module path %q with environment: %q", stacks[0].Name, options.ModulePath, options.Environment) + logging.Logger.Debug().Msgf("found Spacelift stack %q for module path %q with environment: %q", stack.Name, options.ModulePath, options.Environment) vars := map[string]cty.Value{} diff --git a/internal/hcl/remote_variables_loader_test.go b/internal/hcl/remote_variables_loader_test.go index 70f947e8f86..48111cacdab 100644 --- a/internal/hcl/remote_variables_loader_test.go +++ b/internal/hcl/remote_variables_loader_test.go @@ -566,6 +566,7 @@ func TestSpaceliftRemoteVariableLoader_Load(t *testing.T) { }), Metadata: tt.fields.Metadata, cache: &sync.Map{}, + hitMap: &sync.Map{}, } got, err := s.Load(tt.args.options) @@ -581,6 +582,71 @@ func TestSpaceliftRemoteVariableLoader_Load(t *testing.T) { } } +func TestSpaceliftRemoteVariableLoader_Load_ReturnsNilIfAlreadyLoaded(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // This server should only be called once + w.Header().Set("Content-Type", "application/json") + response := `{ + "data": { + "searchStacks": { + "edges": [ + { + "node": { + "id": "module-path-dev", + "name": "module-path:dev", + "projectRoot": "module/path", + "attachedContexts": [ + {"id": "TF_VAR_test", "contextName": "test_value"} + ], + "config": [], + "runtimeConfig": [] + } + } + ], + "pageInfo": { + "endCursor": null, + "hasNextPage": false, + "hasPreviousPage": false + } + } + } + }` + _, _ = w.Write([]byte(response)) + })) + defer ts.Close() + + s := &SpaceliftRemoteVariableLoader{ + Client: client.New(http.DefaultClient, &stubSession{ + token: "test", + endpoint: ts.URL, + }), + Metadata: vcs.Metadata{ + Remote: vcs.Remote{ + Name: "test/test", + }, + }, + cache: &sync.Map{}, + hitMap: &sync.Map{}, + } + + options := RemoteVarLoaderOptions{ + ModulePath: "module/path", + Environment: "dev", + } + + // First call should return variables + got, err := s.Load(options) + assert.NoError(t, err) + assert.Equal(t, map[string]cty.Value{ + "test": cty.StringVal("test_value"), + }, got) + + // Second call should return nil + got, err = s.Load(options) + assert.NoError(t, err) + assert.Nil(t, got) +} + type stubSession struct { token string endpoint string From 39092ac094a5d0372094bdb55be5b483e887ed62 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Thu, 19 Dec 2024 17:30:23 +0000 Subject: [PATCH 67/81] chore: bump go-getter version to support short-hashes --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 456cd97179f..ac9d2f9b810 100644 --- a/go.mod +++ b/go.mod @@ -280,4 +280,4 @@ replace github.com/heimdalr/dag => github.com/aliscott/dag v1.3.2-0.202311151145 replace github.com/shurcooL/graphql => github.com/spacelift-io/graphql v1.2.0 -replace github.com/hashicorp/go-getter => github.com/infracost/go-getter v0.0.0-20240909111353-c0d2eebadfd5 +replace github.com/hashicorp/go-getter => github.com/infracost/go-getter v0.0.0-20241220120307-7fc78d35b563 diff --git a/go.sum b/go.sum index 1b8b541a72b..522f18a3a04 100644 --- a/go.sum +++ b/go.sum @@ -820,8 +820,8 @@ github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/infracost/go-getter v0.0.0-20240909111353-c0d2eebadfd5 h1:FiK2b8h6CezRGI6CGs7YDVG9nbF2TQGMcRK9iSM37so= -github.com/infracost/go-getter v0.0.0-20240909111353-c0d2eebadfd5/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= +github.com/infracost/go-getter v0.0.0-20241220120307-7fc78d35b563 h1:LC9QvTSxATkiPBlFayBZLWe8QY/G8WfrhK/o8voFcDY= +github.com/infracost/go-getter v0.0.0-20241220120307-7fc78d35b563/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= github.com/infracost/terragrunt v0.47.1-0.20241204180911-19aacb9f0819 h1:vh0SyaOOktlG7BPv0aG3VBVOexEyY+H/yONfCX5Tum8= github.com/infracost/terragrunt v0.47.1-0.20241204180911-19aacb9f0819/go.mod h1:504J5iD5AjGgP5IwHkUWAcfoqvZIhrR1aEvyxDxX1VA= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= From c1a58949555a215532c23046f8ed2f8459166f91 Mon Sep 17 00:00:00 2001 From: lewisloofis <15197755+lewisloofis@users.noreply.github.com> Date: Mon, 23 Dec 2024 16:17:46 +0700 Subject: [PATCH 68/81] feat(aws): add aws_cloudfront_function support (#3284) --- .../terraform/aws/cloudfront_function.go | 21 +++++ .../terraform/aws/cloudfront_function_test.go | 16 ++++ internal/providers/terraform/aws/registry.go | 1 + .../cloudfront_function_test.golden | 19 +++++ .../cloudfront_function_test.tf | 37 +++++++++ .../cloudfront_function_test.usage.yml | 4 + internal/resources/aws/cloudfront_function.go | 76 +++++++++++++++++++ 7 files changed, 174 insertions(+) create mode 100644 internal/providers/terraform/aws/cloudfront_function.go create mode 100644 internal/providers/terraform/aws/cloudfront_function_test.go create mode 100644 internal/providers/terraform/aws/testdata/cloudfront_function_test/cloudfront_function_test.golden create mode 100644 internal/providers/terraform/aws/testdata/cloudfront_function_test/cloudfront_function_test.tf create mode 100644 internal/providers/terraform/aws/testdata/cloudfront_function_test/cloudfront_function_test.usage.yml create mode 100644 internal/resources/aws/cloudfront_function.go diff --git a/internal/providers/terraform/aws/cloudfront_function.go b/internal/providers/terraform/aws/cloudfront_function.go new file mode 100644 index 00000000000..7557cdbed2c --- /dev/null +++ b/internal/providers/terraform/aws/cloudfront_function.go @@ -0,0 +1,21 @@ +package aws + +import ( + "github.com/infracost/infracost/internal/resources/aws" + "github.com/infracost/infracost/internal/schema" +) + +func getCloudfrontFunctionRegistryItem() *schema.RegistryItem { + return &schema.RegistryItem{ + Name: "aws_cloudfront_function", + CoreRFunc: newCloudfrontFunction, + } +} + +func newCloudfrontFunction(d *schema.ResourceData) schema.CoreResource { + region := d.Region + return &aws.CloudfrontFunction{ + Address: d.Address, + Region: region, + } +} diff --git a/internal/providers/terraform/aws/cloudfront_function_test.go b/internal/providers/terraform/aws/cloudfront_function_test.go new file mode 100644 index 00000000000..5768f8e715f --- /dev/null +++ b/internal/providers/terraform/aws/cloudfront_function_test.go @@ -0,0 +1,16 @@ +package aws_test + +import ( + "testing" + + "github.com/infracost/infracost/internal/providers/terraform/tftest" +) + +func TestCloudfrontFunction(t *testing.T) { + t.Parallel() + if testing.Short() { + t.Skip("skipping test in short mode") + } + + tftest.GoldenFileResourceTests(t, "cloudfront_function_test") +} diff --git a/internal/providers/terraform/aws/registry.go b/internal/providers/terraform/aws/registry.go index 6af75c741a9..72ba8353e93 100644 --- a/internal/providers/terraform/aws/registry.go +++ b/internal/providers/terraform/aws/registry.go @@ -123,6 +123,7 @@ var ResourceRegistry []*schema.RegistryItem = []*schema.RegistryItem{ getSchedulerScheduleRegistryItem(), getPipesPipeRegistryItem(), getCloudwatchEventTargetRegistryItem(), + getCloudfrontFunctionRegistryItem(), } // FreeResources grouped alphabetically diff --git a/internal/providers/terraform/aws/testdata/cloudfront_function_test/cloudfront_function_test.golden b/internal/providers/terraform/aws/testdata/cloudfront_function_test/cloudfront_function_test.golden new file mode 100644 index 00000000000..f9084ca06e5 --- /dev/null +++ b/internal/providers/terraform/aws/testdata/cloudfront_function_test/cloudfront_function_test.golden @@ -0,0 +1,19 @@ + + Name Monthly Qty Unit Monthly Cost + + aws_cloudfront_function.bibit_deeplink_cf_function + └─ Total number of invocations 0.2 1M invocations $0.02 * + + OVERALL TOTAL $0.02 + +*Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. + +────────────────────────────────── +1 cloud resource was detected: +∙ 1 was estimated + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ +┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ +┃ main ┃ $0.00 ┃ $0.02 ┃ $0.02 ┃ +┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/aws/testdata/cloudfront_function_test/cloudfront_function_test.tf b/internal/providers/terraform/aws/testdata/cloudfront_function_test/cloudfront_function_test.tf new file mode 100644 index 00000000000..3d67587d34a --- /dev/null +++ b/internal/providers/terraform/aws/testdata/cloudfront_function_test/cloudfront_function_test.tf @@ -0,0 +1,37 @@ +provider "aws" { + region = "us-east-1" + skip_credentials_validation = true + skip_metadata_api_check = true + skip_requesting_account_id = true + skip_region_validation = true + access_key = "mock_access_key" + secret_key = "mock_secret_key" +} + +# Add example resources for CloudfrontFunction below + +# resource "aws_cloudfront_function" "cloudfront_function" { +# } +resource "aws_cloudfront_function" "bibit_deeplink_cf_function" { + name = "test_func" + runtime = "cloudfront-js-2.0" + comment = "Bibit Deeplink CF Function Script" + publish = true + code = <<-EOT + async function handler(event) { + var request = event.request; + var uri = request.uri; + + // Check whether the URI is missing a file name. + if (uri.endsWith('/')) { + request.uri += 'index.html'; + } + // Check whether the URI is missing a file extension. + else if (!uri.includes('.')) { + request.uri += '/index.html'; + } + + return request; + } + EOT +} diff --git a/internal/providers/terraform/aws/testdata/cloudfront_function_test/cloudfront_function_test.usage.yml b/internal/providers/terraform/aws/testdata/cloudfront_function_test/cloudfront_function_test.usage.yml new file mode 100644 index 00000000000..c081b3e0069 --- /dev/null +++ b/internal/providers/terraform/aws/testdata/cloudfront_function_test/cloudfront_function_test.usage.yml @@ -0,0 +1,4 @@ +version: 0.1 +resource_usage: + aws_cloudfront_function: + monthly_requests: 200000 diff --git a/internal/resources/aws/cloudfront_function.go b/internal/resources/aws/cloudfront_function.go new file mode 100644 index 00000000000..ff3a6bc400e --- /dev/null +++ b/internal/resources/aws/cloudfront_function.go @@ -0,0 +1,76 @@ +package aws + +import ( + "github.com/infracost/infracost/internal/resources" + "github.com/infracost/infracost/internal/schema" + "github.com/shopspring/decimal" +) + +// CloudfrontFunction struct represents an AWS CloudFront Function. With +// CloudFront Functions, you can write lightweight functions in JavaScript +// for high-scale, latency-sensitive CDN customizations. +// +// Resource information: https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/cloudfront-functions.html +// Pricing information: https://aws.amazon.com/cloudfront/pricing/ +type CloudfrontFunction struct { + Address string + Region string + + MonthlyRequests *int64 `infracost_usage:"monthly_requests"` +} + +// CoreType returns the name of this resource type +func (r *CloudfrontFunction) CoreType() string { + return "CloudfrontFunction" +} + +// UsageSchema defines a list which represents the usage schema of CloudfrontFunction. +func (r *CloudfrontFunction) UsageSchema() []*schema.UsageItem { + return []*schema.UsageItem{ + {Key: "MonthlyRequests", DefaultValue: 0, ValueType: schema.Int64}, + } +} + +// PopulateUsage parses the u schema.UsageData into the CloudfrontFunction. +// It uses the `infracost_usage` struct tags to populate data into the CloudfrontFunction. +func (r *CloudfrontFunction) PopulateUsage(u *schema.UsageData) { + resources.PopulateArgsWithUsage(r, u) +} + +// BuildResource builds a schema.Resource from a valid CloudfrontFunction struct. +// This method is called after the resource is initialised by an IaC provider. +// See providers folder for more information. +func (r *CloudfrontFunction) BuildResource() *schema.Resource { + costComponents := []*schema.CostComponent{} + + costComponents = append(costComponents, r.monthlyRequestsCostComponent()) + + return &schema.Resource{ + Name: r.Address, + UsageSchema: r.UsageSchema(), + CostComponents: costComponents, + } +} + +func (r *CloudfrontFunction) monthlyRequestsCostComponent() *schema.CostComponent { + return &schema.CostComponent{ + Name: "Total number of invocations", + Unit: "1M invocations", + UnitMultiplier: decimal.NewFromInt(1000000), + MonthlyQuantity: intPtrToDecimalPtr(r.MonthlyRequests), + ProductFilter: &schema.ProductFilter{ + VendorName: strPtr("aws"), + Service: strPtr("AmazonCloudFront"), + ProductFamily: strPtr("Request"), + AttributeFilters: []*schema.AttributeFilter{ + {Key: "usagetype", Value: strPtr("Executions-CloudFrontFunctions")}, + {Key: "groupDescription", ValueRegex: regexPtr("CloudFront Function")}, + }, + }, + PriceFilter: &schema.PriceFilter{ + PurchaseOption: strPtr("on_demand"), + StartUsageAmount: strPtr("2000000"), + }, + UsageBased: true, + } +} From 8752c72adf0718ce5e04f821ab0ee1d9c71cbd08 Mon Sep 17 00:00:00 2001 From: Tim McFadden <52185+tim775@users.noreply.github.com> Date: Mon, 23 Dec 2024 10:06:44 -0500 Subject: [PATCH 69/81] fix: correctly apply default tags to aws_instance block devices (#3288) * fix: correctly apply default tags to aws_instance block devices * test: update golden files * test: update golden files --- cmd/infracost/breakdown_test.go | 23 ++ ...n_propagate_defaults_to_volume_tags.golden | 148 +++++++++++++ .../main.tf | 198 ++++++++++++++++++ .../breakdown_format_json_with_tags.golden | 28 +-- ...eakdown_with_policy_data_upload_hcl.golden | 4 +- ...n_with_policy_data_upload_plan_json.golden | 2 +- ..._with_policy_data_upload_terragrunt.golden | 4 +- .../diff_with_policy_data_upload.golden | 2 +- internal/providers/terraform/aws/tags.go | 61 ++++-- .../expected.json | 192 ++++++++--------- .../expected.json | 192 ++++++++--------- 11 files changed, 624 insertions(+), 230 deletions(-) create mode 100644 cmd/infracost/testdata/breakdown_format_json_propagate_defaults_to_volume_tags/breakdown_format_json_propagate_defaults_to_volume_tags.golden create mode 100644 cmd/infracost/testdata/breakdown_format_json_propagate_defaults_to_volume_tags/main.tf diff --git a/cmd/infracost/breakdown_test.go b/cmd/infracost/breakdown_test.go index c1675473bb1..26c01786049 100644 --- a/cmd/infracost/breakdown_test.go +++ b/cmd/infracost/breakdown_test.go @@ -191,6 +191,29 @@ func TestBreakdownFormatJsonWithTagsGoogle(t *testing.T) { ) } +func TestBreakdownFormatJsonPropagateDefaultsToVolumeTags(t *testing.T) { + testName := testutil.CalcGoldenFileTestdataDirName() + dir := path.Join("./testdata", testName) + + GoldenFileCommandTest( + t, + testName, + []string{ + "breakdown", + "--format", "json", + "--path", dir, + }, + &GoldenFileOptions{ + CaptureLogs: true, + IsJSON: true, + JSONInclude: regexp.MustCompile("^(tags|name)$"), + JSONExclude: regexp.MustCompile("^(costComponents|metadata|pastBreakdown|subresources)$"), + }, func(ctx *config.RunContext) { + ctx.Config.TagPoliciesEnabled = true + }, + ) +} + func TestBreakdownFormatJSONShowSkipped(t *testing.T) { opts := DefaultOptions() opts.IsJSON = true diff --git a/cmd/infracost/testdata/breakdown_format_json_propagate_defaults_to_volume_tags/breakdown_format_json_propagate_defaults_to_volume_tags.golden b/cmd/infracost/testdata/breakdown_format_json_propagate_defaults_to_volume_tags/breakdown_format_json_propagate_defaults_to_volume_tags.golden new file mode 100644 index 00000000000..4f75e8cb464 --- /dev/null +++ b/cmd/infracost/testdata/breakdown_format_json_propagate_defaults_to_volume_tags/breakdown_format_json_propagate_defaults_to_volume_tags.golden @@ -0,0 +1,148 @@ +{ + "projects": [ + { + "breakdown": { + "resources": [ + { + "name": "aws_instance.with_root_block_device_no_tags_and_ebs_block_device_no_tags", + "tags": { + "DefaultNoOverride": "default_no_override", + "DefaultOverride": "default_override", + "volume_tags.DefaultNoOverride": "default_no_override", + "volume_tags.DefaultOverride": "default_override" + } + }, + { + "name": "aws_instance.with_root_block_device_no_tags_and_ebs_block_device_no_tags_and_volume_tags", + "tags": { + "DefaultNoOverride": "default_no_override", + "DefaultOverride": "default_override", + "volume_tags.DefaultNoOverride": "default_no_override", + "volume_tags.DefaultOverride": "volume_tag_override", + "volume_tags.VolTag": "volume_tag" + } + }, + { + "name": "aws_instance.with_root_block_device_no_tags_and_ebs_block_device_tags", + "tags": { + "DefaultNoOverride": "default_no_override", + "DefaultOverride": "default_override", + "ebs_block_device[0].DefaultNoOverride": "default_no_override", + "ebs_block_device[0].DefaultOverride": "ebd_tag_override", + "ebs_block_device[0].EBDTagKey": "ebd_tag_val", + "root_block_device.DefaultNoOverride": "default_no_override", + "root_block_device.DefaultOverride": "default_override" + } + }, + { + "name": "aws_instance.with_root_block_device_tags_and_ebs_block_device_no_tags", + "tags": { + "DefaultNoOverride": "default_no_override", + "DefaultOverride": "default_override", + "ebs_block_device[0].DefaultNoOverride": "default_no_override", + "ebs_block_device[0].DefaultOverride": "default_override", + "root_block_device.DefaultNoOverride": "default_no_override", + "root_block_device.DefaultOverride": "rbd_tag_override", + "root_block_device.RBDTagKey": "rbd_tag_val" + } + }, + { + "name": "aws_instance.with_root_block_device_tags_and_ebs_block_device_tags", + "tags": { + "DefaultNoOverride": "default_no_override", + "DefaultOverride": "default_override", + "ebs_block_device[0].DefaultNoOverride": "default_no_override", + "ebs_block_device[0].DefaultOverride": "ebd_tag_override", + "ebs_block_device[0].EBDTagKey": "ebd_tag_val", + "root_block_device.DefaultNoOverride": "default_no_override", + "root_block_device.DefaultOverride": "rbd_tag_override", + "root_block_device.RBDTagKey": "rbd_tag_val" + } + }, + { + "name": "aws_instance.with_implicit_root_block_device_and_ebs_block_device_no_tags", + "tags": { + "DefaultNoOverride": "default_no_override", + "DefaultOverride": "default_override", + "volume_tags.DefaultNoOverride": "default_no_override", + "volume_tags.DefaultOverride": "default_override" + } + }, + { + "name": "aws_instance.with_implicit_root_block_device_and_ebs_block_device_no_tags_and_volume_tags", + "tags": { + "DefaultNoOverride": "default_no_override", + "DefaultOverride": "default_override", + "volume_tags.DefaultNoOverride": "default_no_override", + "volume_tags.DefaultOverride": "volume_tag_override", + "volume_tags.VolTag": "volume_tag" + } + }, + { + "name": "aws_instance.with_implicit_root_block_device_and_ebs_block_device_tags", + "tags": { + "DefaultNoOverride": "default_no_override", + "DefaultOverride": "default_override", + "ebs_block_device[0].DefaultNoOverride": "default_no_override", + "ebs_block_device[0].DefaultOverride": "ebd_tag_override", + "ebs_block_device[0].EBDTagKey": "ebd_tag_val", + "root_block_device.DefaultNoOverride": "default_no_override", + "root_block_device.DefaultOverride": "default_override" + } + }, + { + "name": "aws_instance.with_root_block_device_no_tags", + "tags": { + "DefaultNoOverride": "default_no_override", + "DefaultOverride": "default_override", + "root_block_device.DefaultNoOverride": "default_no_override", + "root_block_device.DefaultOverride": "default_override" + } + }, + { + "name": "aws_instance.with_root_block_device_no_tags_and_volume_tags", + "tags": { + "DefaultNoOverride": "default_no_override", + "DefaultOverride": "default_override", + "volume_tags.DefaultNoOverride": "default_no_override", + "volume_tags.DefaultOverride": "volume_tag_override", + "volume_tags.VolTag": "volume_tag" + } + }, + { + "name": "aws_instance.with_root_block_device_tags", + "tags": { + "DefaultNoOverride": "default_no_override", + "DefaultOverride": "default_override", + "root_block_device.DefaultNoOverride": "default_no_override", + "root_block_device.DefaultOverride": "rbd_tag_override", + "root_block_device.RBDTagKey": "rbd_tag_val" + } + }, + { + "name": "aws_instance.with_implicit_root_block_device", + "tags": { + "DefaultNoOverride": "default_no_override", + "DefaultOverride": "default_override", + "root_block_device.DefaultNoOverride": "default_no_override", + "root_block_device.DefaultOverride": "default_override" + } + }, + { + "name": "aws_instance.with_implicit_root_block_device_and_volume_tags", + "tags": { + "DefaultNoOverride": "default_no_override", + "DefaultOverride": "default_override", + "volume_tags.DefaultNoOverride": "default_no_override", + "volume_tags.DefaultOverride": "volume_tag_override", + "volume_tags.VolTag": "volume_tag" + } + } + ] + }, + "name": "REPLACED_PROJECT_PATH/testdata/breakdown_format_json_propagate_defaults_to_volume_tags" + } + ] +} +Err: + diff --git a/cmd/infracost/testdata/breakdown_format_json_propagate_defaults_to_volume_tags/main.tf b/cmd/infracost/testdata/breakdown_format_json_propagate_defaults_to_volume_tags/main.tf new file mode 100644 index 00000000000..8f21ed10f35 --- /dev/null +++ b/cmd/infracost/testdata/breakdown_format_json_propagate_defaults_to_volume_tags/main.tf @@ -0,0 +1,198 @@ +provider "aws" { + region = "us-east-1" # <<<<< Try changing this to eu-west-1 to compare the costs + skip_credentials_validation = true + skip_requesting_account_id = true + access_key = "mock_access_key" + secret_key = "mock_secret_key" + + default_tags { + tags = { + DefaultNoOverride = "default_no_override" + DefaultOverride = "default_override" + } + } +} + +resource "aws_instance" "with_implicit_root_block_device" { + ami = "ami-674cbc1e" + instance_type = "m5.4xlarge" +} + +resource "aws_instance" "with_implicit_root_block_device_and_volume_tags" { + ami = "ami-674cbc1e" + instance_type = "m5.4xlarge" + + volume_tags = { + VolTag = "volume_tag" + DefaultOverride = "volume_tag_override" + } +} + +resource "aws_instance" "with_implicit_root_block_device_and_ebs_block_device_no_tags" { + ami = "ami-674cbc1e" + instance_type = "m5.4xlarge" + + ebs_block_device { + device_name = "my_data" + volume_size = 50 + } +} + +resource "aws_instance" "with_implicit_root_block_device_and_ebs_block_device_no_tags_and_volume_tags" { + ami = "ami-674cbc1e" + instance_type = "m5.4xlarge" + + ebs_block_device { + device_name = "my_data" + volume_size = 50 + } + + volume_tags = { + VolTag = "volume_tag" + DefaultOverride = "volume_tag_override" + } +} + +resource "aws_instance" "with_implicit_root_block_device_and_ebs_block_device_tags" { + ami = "ami-674cbc1e" + instance_type = "m5.4xlarge" + + ebs_block_device { + device_name = "my_data" + volume_size = 50 + tags = { + EBDTagKey = "ebd_tag_val" + DefaultOverride = "ebd_tag_override" + } + } +} + +resource "aws_instance" "with_root_block_device_no_tags" { + ami = "ami-674cbc1e" + instance_type = "m5.4xlarge" + + root_block_device { + volume_size = 50 + } +} + +resource "aws_instance" "with_root_block_device_no_tags_and_volume_tags" { + ami = "ami-674cbc1e" + instance_type = "m5.4xlarge" + + root_block_device { + volume_size = 50 + } + + volume_tags = { + VolTag = "volume_tag" + DefaultOverride = "volume_tag_override" + } +} + +resource "aws_instance" "with_root_block_device_no_tags_and_ebs_block_device_no_tags" { + ami = "ami-674cbc1e" + instance_type = "m5.4xlarge" + + root_block_device { + volume_size = 50 + } + + ebs_block_device { + device_name = "my_data" + volume_size = 50 + } +} + +resource "aws_instance" "with_root_block_device_no_tags_and_ebs_block_device_no_tags_and_volume_tags" { + ami = "ami-674cbc1e" + instance_type = "m5.4xlarge" + + root_block_device { + volume_size = 50 + } + + ebs_block_device { + device_name = "my_data" + volume_size = 50 + } + + volume_tags = { + VolTag = "volume_tag" + DefaultOverride = "volume_tag_override" + } +} + +resource "aws_instance" "with_root_block_device_no_tags_and_ebs_block_device_tags" { + ami = "ami-674cbc1e" + instance_type = "m5.4xlarge" + + root_block_device { + volume_size = 50 + } + + ebs_block_device { + device_name = "my_data" + volume_size = 50 + tags = { + EBDTagKey = "ebd_tag_val" + DefaultOverride = "ebd_tag_override" + } + } +} + +resource "aws_instance" "with_root_block_device_tags" { + ami = "ami-674cbc1e" + instance_type = "m5.4xlarge" + + root_block_device { + volume_size = 50 + + tags = { + RBDTagKey = "rbd_tag_val" + DefaultOverride = "rbd_tag_override" + } + } +} + +resource "aws_instance" "with_root_block_device_tags_and_ebs_block_device_no_tags" { + ami = "ami-674cbc1e" + instance_type = "m5.4xlarge" + + root_block_device { + volume_size = 50 + + tags = { + RBDTagKey = "rbd_tag_val" + DefaultOverride = "rbd_tag_override" + } + } + + ebs_block_device { + device_name = "my_data" + volume_size = 50 + } +} + +resource "aws_instance" "with_root_block_device_tags_and_ebs_block_device_tags" { + ami = "ami-674cbc1e" + instance_type = "m5.4xlarge" + + root_block_device { + volume_size = 50 + + tags = { + RBDTagKey = "rbd_tag_val" + DefaultOverride = "rbd_tag_override" + } + } + + ebs_block_device { + device_name = "my_data" + volume_size = 50 + tags = { + EBDTagKey = "ebd_tag_val" + DefaultOverride = "ebd_tag_override" + } + } +} diff --git a/cmd/infracost/testdata/breakdown_format_json_with_tags/breakdown_format_json_with_tags.golden b/cmd/infracost/testdata/breakdown_format_json_with_tags/breakdown_format_json_with_tags.golden index 3c3b8f0467a..315404256a4 100644 --- a/cmd/infracost/testdata/breakdown_format_json_with_tags/breakdown_format_json_with_tags.golden +++ b/cmd/infracost/testdata/breakdown_format_json_with_tags/breakdown_format_json_with_tags.golden @@ -61,7 +61,7 @@ } ], "checksum": "3971020356e35708e0de7df3decdd615468f858c38205a99249dd4aa4e3ccf23", - "defaultTagsChecksum": "4758f4f6da3f45b86f7b9d481e2181b234a33f98aca49d98ff65d4f8bc77f24c", + "defaultTagsChecksum": "23ce253ad214e1f28d8a9f8cda886efe35a0731151efe7c37a22652c98fa2101", "endLine": 89, "filename": "REPLACED_PROJECT_PATH/testdata/breakdown_format_json_with_tags/main.tf", "startLine": 67 @@ -135,10 +135,12 @@ "tags": { "DefaultNotOverride": "defaultnotoverride", "DefaultOverride": "defaultoverride", + "ebs_block_device[0].DefaultNotOverride": "defaultnotoverride", + "ebs_block_device[0].DefaultOverride": "defaultoverride", "ebs_block_device[0].foo": "ebd", - "root_block_device.foo": "rbd", - "volume_tags.DefaultNotOverride": "defaultnotoverride", - "volume_tags.DefaultOverride": "defaultoverride" + "root_block_device.DefaultNotOverride": "defaultnotoverride", + "root_block_device.DefaultOverride": "defaultoverride", + "root_block_device.foo": "rbd" }, "defaultTags": { "DefaultNotOverride": "defaultnotoverride", @@ -156,7 +158,7 @@ } ], "checksum": "47efe9c1f904be19ea9a38623ebcb399b03026ba4c0436284a121e35b84ce286", - "defaultTagsChecksum": "4758f4f6da3f45b86f7b9d481e2181b234a33f98aca49d98ff65d4f8bc77f24c", + "defaultTagsChecksum": "23ce253ad214e1f28d8a9f8cda886efe35a0731151efe7c37a22652c98fa2101", "endLine": 203, "filename": "REPLACED_PROJECT_PATH/testdata/breakdown_format_json_with_tags/main.tf", "startLine": 182 @@ -352,7 +354,7 @@ } ], "checksum": "92d793c678fac9bf7ce08e05983f01e62665b7ea640a512f23bb3061b8c645b5", - "defaultTagsChecksum": "4758f4f6da3f45b86f7b9d481e2181b234a33f98aca49d98ff65d4f8bc77f24c", + "defaultTagsChecksum": "23ce253ad214e1f28d8a9f8cda886efe35a0731151efe7c37a22652c98fa2101", "endLine": 103, "filename": "REPLACED_PROJECT_PATH/testdata/breakdown_format_json_with_tags/main.tf", "startLine": 91 @@ -698,7 +700,7 @@ } ], "checksum": "3971020356e35708e0de7df3decdd615468f858c38205a99249dd4aa4e3ccf23", - "defaultTagsChecksum": "4758f4f6da3f45b86f7b9d481e2181b234a33f98aca49d98ff65d4f8bc77f24c", + "defaultTagsChecksum": "23ce253ad214e1f28d8a9f8cda886efe35a0731151efe7c37a22652c98fa2101", "endLine": 89, "filename": "REPLACED_PROJECT_PATH/testdata/breakdown_format_json_with_tags/main.tf", "startLine": 67 @@ -772,10 +774,12 @@ "tags": { "DefaultNotOverride": "defaultnotoverride", "DefaultOverride": "defaultoverride", + "ebs_block_device[0].DefaultNotOverride": "defaultnotoverride", + "ebs_block_device[0].DefaultOverride": "defaultoverride", "ebs_block_device[0].foo": "ebd", - "root_block_device.foo": "rbd", - "volume_tags.DefaultNotOverride": "defaultnotoverride", - "volume_tags.DefaultOverride": "defaultoverride" + "root_block_device.DefaultNotOverride": "defaultnotoverride", + "root_block_device.DefaultOverride": "defaultoverride", + "root_block_device.foo": "rbd" }, "defaultTags": { "DefaultNotOverride": "defaultnotoverride", @@ -793,7 +797,7 @@ } ], "checksum": "47efe9c1f904be19ea9a38623ebcb399b03026ba4c0436284a121e35b84ce286", - "defaultTagsChecksum": "4758f4f6da3f45b86f7b9d481e2181b234a33f98aca49d98ff65d4f8bc77f24c", + "defaultTagsChecksum": "23ce253ad214e1f28d8a9f8cda886efe35a0731151efe7c37a22652c98fa2101", "endLine": 203, "filename": "REPLACED_PROJECT_PATH/testdata/breakdown_format_json_with_tags/main.tf", "startLine": 182 @@ -989,7 +993,7 @@ } ], "checksum": "92d793c678fac9bf7ce08e05983f01e62665b7ea640a512f23bb3061b8c645b5", - "defaultTagsChecksum": "4758f4f6da3f45b86f7b9d481e2181b234a33f98aca49d98ff65d4f8bc77f24c", + "defaultTagsChecksum": "23ce253ad214e1f28d8a9f8cda886efe35a0731151efe7c37a22652c98fa2101", "endLine": 103, "filename": "REPLACED_PROJECT_PATH/testdata/breakdown_format_json_with_tags/main.tf", "startLine": 91 diff --git a/cmd/infracost/testdata/breakdown_with_policy_data_upload_hcl/breakdown_with_policy_data_upload_hcl.golden b/cmd/infracost/testdata/breakdown_with_policy_data_upload_hcl/breakdown_with_policy_data_upload_hcl.golden index 3f633366c1f..c9b764e1f67 100644 --- a/cmd/infracost/testdata/breakdown_with_policy_data_upload_hcl/breakdown_with_policy_data_upload_hcl.golden +++ b/cmd/infracost/testdata/breakdown_with_policy_data_upload_hcl/breakdown_with_policy_data_upload_hcl.golden @@ -63,7 +63,7 @@ } ], "checksum": "f5dd035ad27e97bb0e63f5e9794406f76dec425251f3ad09176905bfe8c1b6c5", - "defaultTagsChecksum": "4758f4f6da3f45b86f7b9d481e2181b234a33f98aca49d98ff65d4f8bc77f24c", + "defaultTagsChecksum": "23ce253ad214e1f28d8a9f8cda886efe35a0731151efe7c37a22652c98fa2101", "endLine": 37, "filename": "REPLACED_PROJECT_PATH/testdata/breakdown_with_policy_data_upload_hcl/main.tf", "startLine": 16 @@ -328,7 +328,7 @@ } ], "checksum": "f5dd035ad27e97bb0e63f5e9794406f76dec425251f3ad09176905bfe8c1b6c5", - "defaultTagsChecksum": "4758f4f6da3f45b86f7b9d481e2181b234a33f98aca49d98ff65d4f8bc77f24c", + "defaultTagsChecksum": "23ce253ad214e1f28d8a9f8cda886efe35a0731151efe7c37a22652c98fa2101", "endLine": 37, "filename": "REPLACED_PROJECT_PATH/testdata/breakdown_with_policy_data_upload_hcl/main.tf", "startLine": 16 diff --git a/cmd/infracost/testdata/breakdown_with_policy_data_upload_plan_json/breakdown_with_policy_data_upload_plan_json.golden b/cmd/infracost/testdata/breakdown_with_policy_data_upload_plan_json/breakdown_with_policy_data_upload_plan_json.golden index 3e300e66192..05c923f8990 100644 --- a/cmd/infracost/testdata/breakdown_with_policy_data_upload_plan_json/breakdown_with_policy_data_upload_plan_json.golden +++ b/cmd/infracost/testdata/breakdown_with_policy_data_upload_plan_json/breakdown_with_policy_data_upload_plan_json.golden @@ -56,7 +56,7 @@ }, "providerSupportsDefaultTags": true, "metadata": { - "defaultTagsChecksum": "4758f4f6da3f45b86f7b9d481e2181b234a33f98aca49d98ff65d4f8bc77f24c" + "defaultTagsChecksum": "23ce253ad214e1f28d8a9f8cda886efe35a0731151efe7c37a22652c98fa2101" }, "hourlyCost": "0.91197260273972602", "monthlyCost": "665.74", diff --git a/cmd/infracost/testdata/breakdown_with_policy_data_upload_terragrunt/breakdown_with_policy_data_upload_terragrunt.golden b/cmd/infracost/testdata/breakdown_with_policy_data_upload_terragrunt/breakdown_with_policy_data_upload_terragrunt.golden index 398e0ad3595..cc6380b1c9c 100644 --- a/cmd/infracost/testdata/breakdown_with_policy_data_upload_terragrunt/breakdown_with_policy_data_upload_terragrunt.golden +++ b/cmd/infracost/testdata/breakdown_with_policy_data_upload_terragrunt/breakdown_with_policy_data_upload_terragrunt.golden @@ -51,7 +51,7 @@ } ], "checksum": "f5dd035ad27e97bb0e63f5e9794406f76dec425251f3ad09176905bfe8c1b6c5", - "defaultTagsChecksum": "4758f4f6da3f45b86f7b9d481e2181b234a33f98aca49d98ff65d4f8bc77f24c", + "defaultTagsChecksum": "23ce253ad214e1f28d8a9f8cda886efe35a0731151efe7c37a22652c98fa2101", "endLine": 22, "filename": "REPLACED_PROJECT_PATH/testdata/breakdown_with_policy_data_upload_terragrunt/main.tf", "startLine": 1 @@ -316,7 +316,7 @@ } ], "checksum": "f5dd035ad27e97bb0e63f5e9794406f76dec425251f3ad09176905bfe8c1b6c5", - "defaultTagsChecksum": "4758f4f6da3f45b86f7b9d481e2181b234a33f98aca49d98ff65d4f8bc77f24c", + "defaultTagsChecksum": "23ce253ad214e1f28d8a9f8cda886efe35a0731151efe7c37a22652c98fa2101", "endLine": 22, "filename": "REPLACED_PROJECT_PATH/testdata/breakdown_with_policy_data_upload_terragrunt/main.tf", "startLine": 1 diff --git a/cmd/infracost/testdata/diff_with_policy_data_upload/diff_with_policy_data_upload.golden b/cmd/infracost/testdata/diff_with_policy_data_upload/diff_with_policy_data_upload.golden index ed72a4bc28a..23a027986fa 100644 --- a/cmd/infracost/testdata/diff_with_policy_data_upload/diff_with_policy_data_upload.golden +++ b/cmd/infracost/testdata/diff_with_policy_data_upload/diff_with_policy_data_upload.golden @@ -147,7 +147,7 @@ } ], "checksum": "f5dd035ad27e97bb0e63f5e9794406f76dec425251f3ad09176905bfe8c1b6c5", - "defaultTagsChecksum": "4758f4f6da3f45b86f7b9d481e2181b234a33f98aca49d98ff65d4f8bc77f24c", + "defaultTagsChecksum": "23ce253ad214e1f28d8a9f8cda886efe35a0731151efe7c37a22652c98fa2101", "endLine": 37, "filename": "REPLACED_PROJECT_PATH/testdata/diff_with_policy_data_upload/main.tf", "startLine": 16 diff --git a/internal/providers/terraform/aws/tags.go b/internal/providers/terraform/aws/tags.go index 4f4b9f0ade5..35fb7b0594f 100644 --- a/internal/providers/terraform/aws/tags.go +++ b/internal/providers/terraform/aws/tags.go @@ -13,7 +13,7 @@ import ( "github.com/infracost/infracost/internal/schema" ) -type parseTagFunc func(baseTags map[string]string, r *schema.ResourceData) +type parseTagFunc func(baseTags, defaultTags map[string]string, r *schema.ResourceData, config TagParsingConfig) var tagProviders = map[string]parseTagFunc{ "aws_instance": parseInstanceTags, @@ -21,7 +21,7 @@ var tagProviders = map[string]parseTagFunc{ "aws_launch_template": parseLaunchTemplateTags, } -func parseLaunchTemplateTags(tags map[string]string, r *schema.ResourceData) { +func parseLaunchTemplateTags(tags, defaultTags map[string]string, r *schema.ResourceData, config TagParsingConfig) { for _, s := range r.Get("tag_specifications").Array() { for k, v := range s.Get("tags").Map() { tags[fmt.Sprintf("tag_specifications.%s", k)] = v.String() @@ -111,13 +111,6 @@ func ParseTags(externalTags, defaultTags map[string]string, r *schema.ResourceDa tags := make(map[string]string) - if r.Type == "aws_instance" && config.PropagateDefaultsToVolumeTags { - for k, v := range defaultTags { - k = fmt.Sprintf("volume_tags.%s", k) - tags[k] = v - } - } - _, supportsDefaultTags := provider_schemas.AWSTagsAllSupport[r.Type] if supportsDefaultTags && defaultTags != nil { keysAndValues := make([]string, 0, len(defaultTags)*2) @@ -126,14 +119,6 @@ func ParseTags(externalTags, defaultTags map[string]string, r *schema.ResourceDa keysAndValues = append(keysAndValues, k, v) } - if r.Type == "aws_instance" && config.PropagateDefaultsToVolumeTags { - for k, v := range defaultTags { - k = fmt.Sprintf("volume_tags.%s", k) - tags[k] = v - keysAndValues = append(keysAndValues, k, v) - } - } - sort.Strings(keysAndValues) h := sha256.New() @@ -178,13 +163,13 @@ func ParseTags(externalTags, defaultTags map[string]string, r *schema.ResourceDa } if f, ok := tagProviders[r.Type]; ok { - f(tags, r) + f(tags, defaultTags, r, config) } return tags, missing } -func parseAutoScalingTags(tags map[string]string, r *schema.ResourceData) { +func parseAutoScalingTags(tags, defaultTags map[string]string, r *schema.ResourceData, config TagParsingConfig) { referencedTagSpecifications(r, func(resourceType string, specs map[string]gjson.Result) { if resourceType == "instance" { for k, v := range specs { @@ -194,7 +179,43 @@ func parseAutoScalingTags(tags map[string]string, r *schema.ResourceData) { }) } -func parseInstanceTags(tags map[string]string, r *schema.ResourceData) { +func parseInstanceTags(tags, defaultTags map[string]string, r *schema.ResourceData, config TagParsingConfig) { + if config.PropagateDefaultsToVolumeTags && len(defaultTags) > 0 { + // when propagating default tags, we add them to volume_tags if they already exist + // or if they cannot conflict with tags defined directly on block devices. To be sure + // there's no conflict: + // 1. there should be no tags defined on any inline root_block_device or ebs_block_device sub resource. + // 2. there must be at least one inline ebs_block_device, because that means there cannot be + // any attached aws_ebs_block_devices. + hasVolTags := r.Get("volume_tags").Exists() + hasRbdTags := r.Get("root_block_device.0.tags").Exists() + hasEbd := false + hasEbdTags := false + for _, ebd := range r.Get("ebs_block_device").Array() { + hasEbd = true + if ebd.Get("tags").Exists() { + hasEbdTags = true + break + } + } + + if hasVolTags || (hasEbd && !hasRbdTags && !hasEbdTags) { + for k, v := range defaultTags { + tags[fmt.Sprintf("volume_tags.%s", k)] = v + } + } else { + // a root_block_device is assumed even if not explicitly defined + for k, v := range defaultTags { + tags[fmt.Sprintf("root_block_device.%s", k)] = v + } + for i := range r.Get("ebs_block_device").Array() { + for k, v := range defaultTags { + tags[fmt.Sprintf("ebs_block_device[%d].%s", i, k)] = v + } + } + } + } + if rbd := r.Get("root_block_device"); rbd.Exists() { for k, v := range rbd.Get("0.tags").Map() { tags[fmt.Sprintf("root_block_device.%s", k)] = v.String() diff --git a/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module/expected.json b/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module/expected.json index 083b0844e43..1415c99a93d 100644 --- a/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module/expected.json +++ b/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module/expected.json @@ -66,15 +66,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_acl.this", - "startLine": 66, - "endLine": 103 + "startLine": 65, + "endLine": 102 } ], "checksum": "2a79b34cee7ece7b7a5231adf7ebbf8efc9b246e0b02c43ff0857bb70742ac53", - "endLine": 103, + "endLine": 102, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 66 + "startLine": 65 } }, { @@ -103,15 +103,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 924, - "endLine": 939 + "startLine": 922, + "endLine": 937 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 939, + "endLine": 937, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 924 + "startLine": 922 } }, { @@ -139,15 +139,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 913, - "endLine": 922 + "startLine": 911, + "endLine": 920 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 922, + "endLine": 920, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 913 + "startLine": 911 } }, { @@ -179,15 +179,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_versioning.this", - "startLine": 160, - "endLine": 174 + "startLine": 159, + "endLine": 173 } ], "checksum": "d87dc101b6296498b9a2114efae840b7b1b1173c2e10fa9b312ae39df0142db5", - "endLine": 174, + "endLine": 173, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 160 + "startLine": 159 } } ], @@ -254,15 +254,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_acl.this", - "startLine": 66, - "endLine": 103 + "startLine": 65, + "endLine": 102 } ], "checksum": "2a79b34cee7ece7b7a5231adf7ebbf8efc9b246e0b02c43ff0857bb70742ac53", - "endLine": 103, + "endLine": 102, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 66 + "startLine": 65 } }, { @@ -291,15 +291,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 924, - "endLine": 939 + "startLine": 922, + "endLine": 937 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 939, + "endLine": 937, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 924 + "startLine": 922 } }, { @@ -327,15 +327,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 913, - "endLine": 922 + "startLine": 911, + "endLine": 920 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 922, + "endLine": 920, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 913 + "startLine": 911 } }, { @@ -367,15 +367,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_versioning.this", - "startLine": 160, - "endLine": 174 + "startLine": 159, + "endLine": 173 } ], "checksum": "d87dc101b6296498b9a2114efae840b7b1b1173c2e10fa9b312ae39df0142db5", - "endLine": 174, + "endLine": 173, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 160 + "startLine": 159 } } ], @@ -507,15 +507,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_acl.this", - "startLine": 66, - "endLine": 103 + "startLine": 65, + "endLine": 102 } ], "checksum": "2a79b34cee7ece7b7a5231adf7ebbf8efc9b246e0b02c43ff0857bb70742ac53", - "endLine": 103, + "endLine": 102, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 66 + "startLine": 65 } }, { @@ -544,15 +544,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 924, - "endLine": 939 + "startLine": 922, + "endLine": 937 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 939, + "endLine": 937, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 924 + "startLine": 922 } }, { @@ -580,15 +580,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 913, - "endLine": 922 + "startLine": 911, + "endLine": 920 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 922, + "endLine": 920, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 913 + "startLine": 911 } }, { @@ -620,15 +620,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_versioning.this", - "startLine": 160, - "endLine": 174 + "startLine": 159, + "endLine": 173 } ], "checksum": "d87dc101b6296498b9a2114efae840b7b1b1173c2e10fa9b312ae39df0142db5", - "endLine": 174, + "endLine": 173, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 160 + "startLine": 159 } } ], @@ -890,15 +890,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_acl.this", - "startLine": 66, - "endLine": 103 + "startLine": 65, + "endLine": 102 } ], "checksum": "2a79b34cee7ece7b7a5231adf7ebbf8efc9b246e0b02c43ff0857bb70742ac53", - "endLine": 103, + "endLine": 102, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 66 + "startLine": 65 } }, { @@ -927,15 +927,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 924, - "endLine": 939 + "startLine": 922, + "endLine": 937 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 939, + "endLine": 937, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 924 + "startLine": 922 } }, { @@ -963,15 +963,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 913, - "endLine": 922 + "startLine": 911, + "endLine": 920 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 922, + "endLine": 920, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 913 + "startLine": 911 } }, { @@ -1003,15 +1003,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_versioning.this", - "startLine": 160, - "endLine": 174 + "startLine": 159, + "endLine": 173 } ], "checksum": "d87dc101b6296498b9a2114efae840b7b1b1173c2e10fa9b312ae39df0142db5", - "endLine": 174, + "endLine": 173, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 160 + "startLine": 159 } } ], @@ -1078,15 +1078,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_acl.this", - "startLine": 66, - "endLine": 103 + "startLine": 65, + "endLine": 102 } ], "checksum": "2a79b34cee7ece7b7a5231adf7ebbf8efc9b246e0b02c43ff0857bb70742ac53", - "endLine": 103, + "endLine": 102, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 66 + "startLine": 65 } }, { @@ -1115,15 +1115,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 924, - "endLine": 939 + "startLine": 922, + "endLine": 937 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 939, + "endLine": 937, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 924 + "startLine": 922 } }, { @@ -1151,15 +1151,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 913, - "endLine": 922 + "startLine": 911, + "endLine": 920 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 922, + "endLine": 920, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 913 + "startLine": 911 } }, { @@ -1191,15 +1191,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_versioning.this", - "startLine": 160, - "endLine": 174 + "startLine": 159, + "endLine": 173 } ], "checksum": "d87dc101b6296498b9a2114efae840b7b1b1173c2e10fa9b312ae39df0142db5", - "endLine": 174, + "endLine": 173, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 160 + "startLine": 159 } } ], @@ -1331,15 +1331,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_acl.this", - "startLine": 66, - "endLine": 103 + "startLine": 65, + "endLine": 102 } ], "checksum": "2a79b34cee7ece7b7a5231adf7ebbf8efc9b246e0b02c43ff0857bb70742ac53", - "endLine": 103, + "endLine": 102, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 66 + "startLine": 65 } }, { @@ -1368,15 +1368,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 924, - "endLine": 939 + "startLine": 922, + "endLine": 937 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 939, + "endLine": 937, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 924 + "startLine": 922 } }, { @@ -1404,15 +1404,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 913, - "endLine": 922 + "startLine": 911, + "endLine": 920 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 922, + "endLine": 920, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 913 + "startLine": 911 } }, { @@ -1444,15 +1444,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_versioning.this", - "startLine": 160, - "endLine": 174 + "startLine": 159, + "endLine": 173 } ], "checksum": "d87dc101b6296498b9a2114efae840b7b1b1173c2e10fa9b312ae39df0142db5", - "endLine": 174, + "endLine": 173, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 160 + "startLine": 159 } } ], diff --git a/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module_chdir/expected.json b/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module_chdir/expected.json index 58272378b92..07d230b50f2 100644 --- a/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module_chdir/expected.json +++ b/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module_chdir/expected.json @@ -66,15 +66,15 @@ { "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_acl.this", - "startLine": 66, - "endLine": 103 + "startLine": 65, + "endLine": 102 } ], "checksum": "2a79b34cee7ece7b7a5231adf7ebbf8efc9b246e0b02c43ff0857bb70742ac53", - "endLine": 103, + "endLine": 102, "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 66 + "startLine": 65 } }, { @@ -103,15 +103,15 @@ { "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 924, - "endLine": 939 + "startLine": 922, + "endLine": 937 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 939, + "endLine": 937, "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 924 + "startLine": 922 } }, { @@ -139,15 +139,15 @@ { "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 913, - "endLine": 922 + "startLine": 911, + "endLine": 920 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 922, + "endLine": 920, "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 913 + "startLine": 911 } }, { @@ -179,15 +179,15 @@ { "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_versioning.this", - "startLine": 160, - "endLine": 174 + "startLine": 159, + "endLine": 173 } ], "checksum": "d87dc101b6296498b9a2114efae840b7b1b1173c2e10fa9b312ae39df0142db5", - "endLine": 174, + "endLine": 173, "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 160 + "startLine": 159 } } ], @@ -254,15 +254,15 @@ { "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_acl.this", - "startLine": 66, - "endLine": 103 + "startLine": 65, + "endLine": 102 } ], "checksum": "2a79b34cee7ece7b7a5231adf7ebbf8efc9b246e0b02c43ff0857bb70742ac53", - "endLine": 103, + "endLine": 102, "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 66 + "startLine": 65 } }, { @@ -291,15 +291,15 @@ { "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 924, - "endLine": 939 + "startLine": 922, + "endLine": 937 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 939, + "endLine": 937, "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 924 + "startLine": 922 } }, { @@ -327,15 +327,15 @@ { "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 913, - "endLine": 922 + "startLine": 911, + "endLine": 920 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 922, + "endLine": 920, "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 913 + "startLine": 911 } }, { @@ -367,15 +367,15 @@ { "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_versioning.this", - "startLine": 160, - "endLine": 174 + "startLine": 159, + "endLine": 173 } ], "checksum": "d87dc101b6296498b9a2114efae840b7b1b1173c2e10fa9b312ae39df0142db5", - "endLine": 174, + "endLine": 173, "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 160 + "startLine": 159 } } ], @@ -507,15 +507,15 @@ { "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_acl.this", - "startLine": 66, - "endLine": 103 + "startLine": 65, + "endLine": 102 } ], "checksum": "2a79b34cee7ece7b7a5231adf7ebbf8efc9b246e0b02c43ff0857bb70742ac53", - "endLine": 103, + "endLine": 102, "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 66 + "startLine": 65 } }, { @@ -544,15 +544,15 @@ { "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 924, - "endLine": 939 + "startLine": 922, + "endLine": 937 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 939, + "endLine": 937, "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 924 + "startLine": 922 } }, { @@ -580,15 +580,15 @@ { "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 913, - "endLine": 922 + "startLine": 911, + "endLine": 920 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 922, + "endLine": 920, "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 913 + "startLine": 911 } }, { @@ -620,15 +620,15 @@ { "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_versioning.this", - "startLine": 160, - "endLine": 174 + "startLine": 159, + "endLine": 173 } ], "checksum": "d87dc101b6296498b9a2114efae840b7b1b1173c2e10fa9b312ae39df0142db5", - "endLine": 174, + "endLine": 173, "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 160 + "startLine": 159 } } ], @@ -890,15 +890,15 @@ { "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_acl.this", - "startLine": 66, - "endLine": 103 + "startLine": 65, + "endLine": 102 } ], "checksum": "2a79b34cee7ece7b7a5231adf7ebbf8efc9b246e0b02c43ff0857bb70742ac53", - "endLine": 103, + "endLine": 102, "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 66 + "startLine": 65 } }, { @@ -927,15 +927,15 @@ { "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 924, - "endLine": 939 + "startLine": 922, + "endLine": 937 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 939, + "endLine": 937, "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 924 + "startLine": 922 } }, { @@ -963,15 +963,15 @@ { "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 913, - "endLine": 922 + "startLine": 911, + "endLine": 920 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 922, + "endLine": 920, "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 913 + "startLine": 911 } }, { @@ -1003,15 +1003,15 @@ { "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "blockName": "aws_s3_bucket_versioning.this", - "startLine": 160, - "endLine": 174 + "startLine": 159, + "endLine": 173 } ], "checksum": "d87dc101b6296498b9a2114efae840b7b1b1173c2e10fa9b312ae39df0142db5", - "endLine": 174, + "endLine": 173, "filename": ".infracost/terraform_modules/30b695ab2291e2ded2ac2e47e63b88e6/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 160 + "startLine": 159 } } ], @@ -1078,15 +1078,15 @@ { "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_acl.this", - "startLine": 66, - "endLine": 103 + "startLine": 65, + "endLine": 102 } ], "checksum": "2a79b34cee7ece7b7a5231adf7ebbf8efc9b246e0b02c43ff0857bb70742ac53", - "endLine": 103, + "endLine": 102, "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 66 + "startLine": 65 } }, { @@ -1115,15 +1115,15 @@ { "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 924, - "endLine": 939 + "startLine": 922, + "endLine": 937 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 939, + "endLine": 937, "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 924 + "startLine": 922 } }, { @@ -1151,15 +1151,15 @@ { "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 913, - "endLine": 922 + "startLine": 911, + "endLine": 920 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 922, + "endLine": 920, "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 913 + "startLine": 911 } }, { @@ -1191,15 +1191,15 @@ { "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_versioning.this", - "startLine": 160, - "endLine": 174 + "startLine": 159, + "endLine": 173 } ], "checksum": "d87dc101b6296498b9a2114efae840b7b1b1173c2e10fa9b312ae39df0142db5", - "endLine": 174, + "endLine": 173, "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 160 + "startLine": 159 } } ], @@ -1331,15 +1331,15 @@ { "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_acl.this", - "startLine": 66, - "endLine": 103 + "startLine": 65, + "endLine": 102 } ], "checksum": "2a79b34cee7ece7b7a5231adf7ebbf8efc9b246e0b02c43ff0857bb70742ac53", - "endLine": 103, + "endLine": 102, "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 66 + "startLine": 65 } }, { @@ -1368,15 +1368,15 @@ { "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 924, - "endLine": 939 + "startLine": 922, + "endLine": 937 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 939, + "endLine": 937, "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 924 + "startLine": 922 } }, { @@ -1404,15 +1404,15 @@ { "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 913, - "endLine": 922 + "startLine": 911, + "endLine": 920 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 922, + "endLine": 920, "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 913 + "startLine": 911 } }, { @@ -1444,15 +1444,15 @@ { "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "blockName": "aws_s3_bucket_versioning.this", - "startLine": 160, - "endLine": 174 + "startLine": 159, + "endLine": 173 } ], "checksum": "d87dc101b6296498b9a2114efae840b7b1b1173c2e10fa9b312ae39df0142db5", - "endLine": 174, + "endLine": 173, "filename": ".infracost/terraform_modules/32fe3779b4dcd7ba3d9f5ca25112691d/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/HEAD/main.tf", - "startLine": 160 + "startLine": 159 } } ], From 2db4c156d0004d74b202bd6d2f10377d6762a7ca Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Fri, 20 Dec 2024 16:55:12 +0000 Subject: [PATCH 70/81] enhance: update URL normalizing to work with both SSH and HTTPS URLs * It now works with converting between Azure Devops SSH URLs to HTTPS URLS * It now strips users from HTTPS URLs * There's shared logic between using it for forcing HTTPS downloads and generating module HTTPS URLs --- internal/hcl/modules/fetch.go | 52 +++++++++++------ internal/hcl/modules/fetch_test.go | 19 +++++-- internal/providers/terraform/hcl_provider.go | 56 +++++++++---------- .../providers/terraform/hcl_provider_test.go | 27 +++++---- .../terraform/terragrunt_hcl_provider.go | 2 +- 5 files changed, 89 insertions(+), 67 deletions(-) diff --git a/internal/hcl/modules/fetch.go b/internal/hcl/modules/fetch.go index d276a4625fa..7a52c612587 100644 --- a/internal/hcl/modules/fetch.go +++ b/internal/hcl/modules/fetch.go @@ -261,7 +261,7 @@ func (g *CustomGitGetter) Get(dst string, u *url.URL) error { return g.GitGetter.Get(dst, u) } - httpsURL, err := TransformSSHToHttps(u) + httpsURL, err := NormalizeGitURLToHTTPS(u) if err != nil { logging.Logger.Debug().Err(err).Msgf("failed to transform %s to https", u) return g.GitGetter.Get(dst, u) @@ -284,35 +284,51 @@ func IsGitSSHSource(u *url.URL) bool { return false } - if u.Scheme == "ssh" || u.Scheme == "git::ssh" { + if u.Scheme == "ssh" || u.Scheme == "git::ssh" || u.Scheme == "git+ssh" { return true } return false } -// TransformSSHToHttps transforms a Terraform module source url to an HTTPS -// equivalent. This only handles source urls prefixed with ssh:: or git::ssh. The -// shorthand ssh source referenced here: -// https://developer.hashicorp.com/terraform/language/modules/sources#github e.g. -// "git@github.com:hashicorp/example.git" in not handled by this method as we -// expect the source to already be Detected to the valid longhand equivalent -// before calling this function. This can be achieved by calling -// getter.Detect(src) before calling TransformSSHToHttps. -func TransformSSHToHttps(u *url.URL) (*url.URL, error) { - if !IsGitSSHSource(u) { - return u, nil +// NormalizeGitURLToHTTPS normalizes a Git source url to an HTTPS equivalent. +// It supports SSH URLs, as well as HTTPS URLS with usernames. +// This allows us to convert Terraform module source urls to HTTPS URLs, so we can +// attempt to download over HTTPS first using the existing credentials we have. +// It can also be used for generating links to resources within the module. +// There is a special case for Azure DevOps SSH URLs to handle converting them +// to the equivalent HTTPS URL. +func NormalizeGitURLToHTTPS(u *url.URL) (*url.URL, error) { + hostname := u.Host + // Strip the port if it's an SSH url + if IsGitSSHSource(u) { + hostname = strings.Split(hostname, ":")[0] } - hostname := u.Host path := strings.TrimPrefix(u.Path, "/") - // SSH URLs might contain ':' after the host (like 'git@hostname:user/repo.git') - // We need to replace the first ':' with a '/' - if idx := strings.Index(path, ":"); idx != -1 { - path = path[:idx] + "/" + path[idx+1:] + // Handle Azure DevOps SSH URLs + if hostname == "ssh.dev.azure.com" { + // Azure DevOps URLs need special handling + // Convert from: ssh://git@ssh.dev.azure.com/v3/org/project/repo + // To: https://dev.azure.com/org/project/_git/repo + parts := strings.Split(path, "/") + if len(parts) >= 4 && strings.HasPrefix(parts[0], "v") { + org := parts[1] + project := parts[2] + repo := parts[3] + return &url.URL{ + Scheme: "https", + Host: "dev.azure.com", + Path: fmt.Sprintf("/%s/%s/_git/%s", org, project, repo), + }, nil + } + return nil, fmt.Errorf("invalid Azure DevOps SSH URL format") } + // Strip .git from the end of the path + path = strings.TrimSuffix(path, ".git") + return &url.URL{ Scheme: "https", Host: hostname, diff --git a/internal/hcl/modules/fetch_test.go b/internal/hcl/modules/fetch_test.go index eafa174e342..8810fe783d1 100644 --- a/internal/hcl/modules/fetch_test.go +++ b/internal/hcl/modules/fetch_test.go @@ -16,20 +16,27 @@ import ( "github.com/stretchr/testify/require" ) -func TestTransformSSHToHTTPS(t *testing.T) { +func TestNormalizeGitURLToHTTPS(t *testing.T) { testCases := []struct { sshURL *url.URL expected string }{ - {&url.URL{Scheme: "ssh", User: url.User("git"), Path: "user/repo.git", Host: "github.com"}, "https://github.com/user/repo.git"}, - {&url.URL{Scheme: "https", Path: "user/repo.git", Host: "github.com"}, "https://github.com/user/repo.git"}, - {&url.URL{Scheme: "git::ssh", User: url.User("git"), Path: "user/repo.git", Host: "github.com"}, "https://github.com/user/repo.git"}, + {&url.URL{Scheme: "ssh", User: url.User("git"), Path: "user/repo.git", Host: "github.com"}, "https://github.com/user/repo"}, + {&url.URL{Scheme: "ssh", User: url.User("git"), Path: "group/project.git", Host: "gitlab.com"}, "https://gitlab.com/group/project"}, + {&url.URL{Scheme: "ssh", User: url.User("git"), Path: "team/repo.git", Host: "bitbucket.org"}, "https://bitbucket.org/team/repo"}, + {&url.URL{Scheme: "ssh", User: url.User("git"), Path: "v3/organization/project/repo", Host: "ssh.dev.azure.com"}, "https://dev.azure.com/organization/project/_git/repo"}, // Azure Repos + {&url.URL{Scheme: "ssh", User: url.User("user"), Path: "user/repo.git", Host: "github.com"}, "https://github.com/user/repo"}, // ssh with custom username + {&url.URL{Scheme: "ssh", User: url.User("git"), Path: "/user/repo.git", Host: "myserver.com:2222"}, "https://myserver.com/user/repo"}, // with port + {&url.URL{Scheme: "git+ssh", User: url.User("git"), Path: "/user/repo.git", Host: "myserver.com"}, "https://myserver.com/user/repo"}, // with git+ssh + {&url.URL{Scheme: "git::ssh", User: url.User("git"), Path: "/user/repo.git", Host: "myserver.com"}, "https://myserver.com/user/repo"}, // with git::ssh + {&url.URL{Scheme: "https", Path: "/user/repo.git", Host: "github.com"}, "https://github.com/user/repo"}, // https + {&url.URL{Scheme: "https", User: url.User("user"), Path: "/user/repo.git", Host: "myserver.com"}, "https://myserver.com/user/repo"}, // https with username + {&url.URL{Scheme: "git::https", Path: "/user/repo.git", Host: "github.com"}, "https://github.com/user/repo"}, // git::https } for _, tc := range testCases { - transformed, err := TransformSSHToHttps(tc.sshURL) + transformed, err := NormalizeGitURLToHTTPS(tc.sshURL) assert.NoError(t, err) - assert.Equal(t, tc.expected, transformed.String()) } } diff --git a/internal/providers/terraform/hcl_provider.go b/internal/providers/terraform/hcl_provider.go index 296f750ee60..6ee01b0162a 100644 --- a/internal/providers/terraform/hcl_provider.go +++ b/internal/providers/terraform/hcl_provider.go @@ -15,6 +15,7 @@ import ( "sort" "strings" + giturls "github.com/chainguard-dev/git-urls" "github.com/infracost/infracost/internal/metrics" "github.com/infracost/infracost/internal/schema" @@ -603,7 +604,7 @@ func (p *HCLProvider) getResourceOutput(block *hcl.Block, moduleSourceURL string } func buildModuleFilename(filename string, moduleSourceURL string) string { - httpsURL, err := transformSSHToHTTPS(moduleSourceURL) + httpsURL, err := normalizeModuleURL(moduleSourceURL) if err != nil { logging.Logger.Debug().Err(err).Msgf("failed to build module filename, could not transform url %s to https", moduleSourceURL) return "" @@ -628,42 +629,35 @@ func buildModuleFilename(filename string, moduleSourceURL string) string { return moduleFilename } -func transformSSHToHTTPS(sshURL string) (string, error) { - if !strings.HasPrefix(sshURL, "git@") { - // nothing to do, the URL is not an SSH URL - return sshURL, nil +func normalizeModuleURL(sshURL string) (string, error) { + // git::ssh and git:https aren't recognized as a valid URL scheme, so we need to strip it so they just use ssh or https schemes + u := sshURL + if strings.HasPrefix(sshURL, "git::") { + u = strings.Replace(sshURL, "git::", "", 1) } - colonCount := strings.Count(sshURL, ":") - var domainAndPort, path string - - switch colonCount { - case 1: - components := strings.SplitN(sshURL, ":", 2) - userAndDomain := strings.Split(components[0], "@") - if len(userAndDomain) != 2 { - return "", fmt.Errorf("invalid SSH URL format") - } - domainAndPort = userAndDomain[1] - path = components[1] - case 2: - // case with a port in the url - components := strings.SplitN(sshURL, ":", 3) - userAndDomain := strings.Split(components[0], "@") - if len(userAndDomain) != 2 { - return "", fmt.Errorf("invalid SSH URL format") + parsedURL, err := url.Parse(u) + if err != nil { + // Try parsing it as a git URL + parsedURL, err = giturls.Parse(u) + if err != nil { + return "", err } - domainAndPort = userAndDomain[1] - path = components[2] - default: - return "", fmt.Errorf("invalid SSH URL format") } - // remove port if it exists - domainComponents := strings.SplitN(domainAndPort, ":", 2) - domain := domainComponents[0] + // Save the query string, since this is lost when we normalize the URL + // and it may contain the ref + query := parsedURL.Query() + + res, err := modules.NormalizeGitURLToHTTPS(parsedURL) + if err != nil { + return "", err + } + + // Add the query string back in + res.RawQuery = query.Encode() - return strings.TrimSuffix(fmt.Sprintf("https://%s/%s", domain, path), "/"), nil + return res.String(), nil } func (p *HCLProvider) marshalProviderBlock(block *hcl.Block) string { diff --git a/internal/providers/terraform/hcl_provider_test.go b/internal/providers/terraform/hcl_provider_test.go index 3ccd2f52a22..40360d10310 100644 --- a/internal/providers/terraform/hcl_provider_test.go +++ b/internal/providers/terraform/hcl_provider_test.go @@ -292,29 +292,34 @@ func TestHCLProvider_LoadPlanJSON(t *testing.T) { } } -func TestTransformSSHToHTTPS(t *testing.T) { +func TestNormalizeModuleURL(t *testing.T) { tests := []struct { input string expected string }{ - {"git@github.com:user/repo.git", "https://github.com/user/repo.git"}, - {"git@gitlab.com:group/project.git", "https://gitlab.com/group/project.git"}, - {"git@bitbucket.org:team/repo.git", "https://bitbucket.org/team/repo.git"}, - {"git@myserver.com:2222:user/repo.git", "https://myserver.com/user/repo.git"}, // with port - {"git@ssh.dev.azure.com:v3/organization/project/repo", "https://ssh.dev.azure.com/v3/organization/project/repo"}, // Azure Repos - {"invalid-url", "invalid-url"}, // invalid SSH URL - {"user@github.com:user/repo.git", "user@github.com:user/repo.git"}, // unexpected username + {"git@github.com:user/repo.git", "https://github.com/user/repo"}, + {"git@gitlab.com:group/project.git", "https://gitlab.com/group/project"}, + {"git@bitbucket.org:team/repo.git", "https://bitbucket.org/team/repo"}, + {"git@ssh.dev.azure.com:v3/organization/project/repo", "https://dev.azure.com/organization/project/_git/repo"}, // Azure Repos + {"user@github.com:user/repo.git", "https://github.com/user/repo"}, // ssh with custom username + {"ssh://git@myserver.com:2222/user/repo.git", "https://myserver.com/user/repo"}, // with port + {"git+ssh://git@myserver.com/user/repo.git", "https://myserver.com/user/repo"}, // with git+ssh + {"git::ssh://git@myserver.com/user/repo.git", "https://myserver.com/user/repo"}, // with git::ssh + {"https://github.com/user/repo.git", "https://github.com/user/repo"}, // https + {"https://user@myserver.com/user/repo.git", "https://myserver.com/user/repo"}, // https with username + {"git::https://github.com/user/repo.git", "https://github.com/user/repo"}, // git::https + {"git::ssh://git@myserver.com/user/repo.git?ref=a094835", "https://myserver.com/user/repo?ref=a094835"}, // git::ssh with ref } for _, test := range tests { - output, err := transformSSHToHTTPS(test.input) + output, err := normalizeModuleURL(test.input) if err != nil { if test.expected != "" { - t.Errorf("transformSSHToHTTPS(%q) returned an error: %v", test.input, err) + t.Errorf("normalizeModuleURL(%q) returned an error: %v", test.input, err) } } else { if !reflect.DeepEqual(output, test.expected) { - t.Errorf("transformSSHToHTTPS(%q) = %q, want %q", test.input, output, test.expected) + t.Errorf("normalizeModuleURL(%q) = %q, want %q", test.input, output, test.expected) } } } diff --git a/internal/providers/terraform/terragrunt_hcl_provider.go b/internal/providers/terraform/terragrunt_hcl_provider.go index cee93d0bfe5..ad7d0f691a2 100644 --- a/internal/providers/terraform/terragrunt_hcl_provider.go +++ b/internal/providers/terraform/terragrunt_hcl_provider.go @@ -803,7 +803,7 @@ func forceHttpsDownload(sourceURL string, opts *tgoptions.TerragruntOptions, ter return false } - newUrl, err := modules.TransformSSHToHttps(u) + newUrl, err := modules.NormalizeGitURLToHTTPS(u) if err != nil { return false } From e633f8d8101e0767ae7c5fa293684f3c6d7edea8 Mon Sep 17 00:00:00 2001 From: Tim McFadden <52185+tim775@users.noreply.github.com> Date: Mon, 23 Dec 2024 13:16:15 -0500 Subject: [PATCH 71/81] chore: make tagschema (#3289) --- internal/providers/terraform/provider_schemas/aws.tags.json | 1 + internal/providers/terraform/provider_schemas/aws.tags_all.json | 1 + 2 files changed, 2 insertions(+) diff --git a/internal/providers/terraform/provider_schemas/aws.tags.json b/internal/providers/terraform/provider_schemas/aws.tags.json index 76aeaddf843..f137b5c8cdc 100644 --- a/internal/providers/terraform/provider_schemas/aws.tags.json +++ b/internal/providers/terraform/provider_schemas/aws.tags.json @@ -510,6 +510,7 @@ "aws_redshiftserverless_namespace": true, "aws_redshiftserverless_workgroup": true, "aws_rekognition_collection": true, + "aws_rekognition_stream_processor": true, "aws_resourceexplorer2_index": true, "aws_resourceexplorer2_view": true, "aws_resourcegroups_group": true, diff --git a/internal/providers/terraform/provider_schemas/aws.tags_all.json b/internal/providers/terraform/provider_schemas/aws.tags_all.json index ea498ae76ce..daeb858991b 100644 --- a/internal/providers/terraform/provider_schemas/aws.tags_all.json +++ b/internal/providers/terraform/provider_schemas/aws.tags_all.json @@ -509,6 +509,7 @@ "aws_redshiftserverless_namespace": true, "aws_redshiftserverless_workgroup": true, "aws_rekognition_collection": true, + "aws_rekognition_stream_processor": true, "aws_resourceexplorer2_index": true, "aws_resourceexplorer2_view": true, "aws_resourcegroups_group": true, From 859f2ca9f4174e17528a417776768d751778d3f8 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Tue, 24 Dec 2024 08:53:15 +0000 Subject: [PATCH 72/81] fix: ensure we try normalized HTTPs URLS when downloading modules --- internal/hcl/modules/fetch.go | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/internal/hcl/modules/fetch.go b/internal/hcl/modules/fetch.go index 7a52c612587..a99fd473ce9 100644 --- a/internal/hcl/modules/fetch.go +++ b/internal/hcl/modules/fetch.go @@ -245,22 +245,19 @@ func determineTTL(moduleAddr string) time.Duration { return defaultTTL } -// CustomGitGetter extends the standard GitGetter transforming SSH sources to -// HTTPs first before attempting a Get. This means that we can attempt to use any -// Git credentials on the host machine to resolve the Get before falling back to -// SSH. +// CustomGitGetter extends the standard GitGetter and normalizes SSH and HTTPS URLs +// so it can attempt to use the Git credentials on the host machine to resolve the +// Get before falling back to the original method. +// SSH URLs are transformed to their HTTPS equivalent before attempting a Get. +// HTTPS URLS are stripped of any credentials. type CustomGitGetter struct { *getter.GitGetter } -// Get overrides the standard Get method transforming SSH urls to their HTTPS -// equivalent. Get then tries to get the new url into the dst, falling back to -// the original SSH url if an HTTPS get fails. +// Get overrides the standard Get method to normalize SSH and HTTPS URLs before +// attempting a Get. If the normalized URL fails it falls back to the original +// URL. func (g *CustomGitGetter) Get(dst string, u *url.URL) error { - if u.Scheme != "ssh" { - return g.GitGetter.Get(dst, u) - } - httpsURL, err := NormalizeGitURLToHTTPS(u) if err != nil { logging.Logger.Debug().Err(err).Msgf("failed to transform %s to https", u) @@ -330,8 +327,10 @@ func NormalizeGitURLToHTTPS(u *url.URL) (*url.URL, error) { path = strings.TrimSuffix(path, ".git") return &url.URL{ - Scheme: "https", - Host: hostname, - Path: path, + Scheme: "https", + Host: hostname, + Path: path, + RawQuery: u.RawQuery, + RawFragment: u.RawFragment, }, nil } From 4f209de1b00af8c7ca309e584be5c6cd7232fbbc Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Tue, 24 Dec 2024 09:23:41 +0000 Subject: [PATCH 73/81] chore: update golden test file lines --- .../expected.json | 64 +++++++++---------- .../expected.json | 64 +++++++++---------- 2 files changed, 64 insertions(+), 64 deletions(-) diff --git a/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module/expected.json b/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module/expected.json index 1415c99a93d..22f1421b6fa 100644 --- a/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module/expected.json +++ b/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module/expected.json @@ -254,15 +254,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_acl.this", - "startLine": 65, - "endLine": 102 + "startLine": 66, + "endLine": 103 } ], "checksum": "2a79b34cee7ece7b7a5231adf7ebbf8efc9b246e0b02c43ff0857bb70742ac53", - "endLine": 102, + "endLine": 103, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 65 + "startLine": 66 } }, { @@ -291,15 +291,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 922, - "endLine": 937 + "startLine": 923, + "endLine": 938 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 937, + "endLine": 938, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 922 + "startLine": 923 } }, { @@ -327,15 +327,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 911, - "endLine": 920 + "startLine": 912, + "endLine": 921 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 920, + "endLine": 921, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 911 + "startLine": 912 } }, { @@ -367,15 +367,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_versioning.this", - "startLine": 159, - "endLine": 173 + "startLine": 160, + "endLine": 174 } ], "checksum": "d87dc101b6296498b9a2114efae840b7b1b1173c2e10fa9b312ae39df0142db5", - "endLine": 173, + "endLine": 174, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 159 + "startLine": 160 } } ], @@ -1078,15 +1078,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_acl.this", - "startLine": 65, - "endLine": 102 + "startLine": 66, + "endLine": 103 } ], "checksum": "2a79b34cee7ece7b7a5231adf7ebbf8efc9b246e0b02c43ff0857bb70742ac53", - "endLine": 102, + "endLine": 103, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 65 + "startLine": 66 } }, { @@ -1115,15 +1115,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 922, - "endLine": 937 + "startLine": 923, + "endLine": 938 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 937, + "endLine": 938, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 922 + "startLine": 923 } }, { @@ -1151,15 +1151,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 911, - "endLine": 920 + "startLine": 912, + "endLine": 921 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 920, + "endLine": 921, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 911 + "startLine": 912 } }, { @@ -1191,15 +1191,15 @@ { "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_versioning.this", - "startLine": 159, - "endLine": 173 + "startLine": 160, + "endLine": 174 } ], "checksum": "d87dc101b6296498b9a2114efae840b7b1b1173c2e10fa9b312ae39df0142db5", - "endLine": 173, + "endLine": 174, "filename": "testdata/hcl_provider_test/adds_source_url_from_remote_module/.infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 159 + "startLine": 160 } } ], diff --git a/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module_chdir/expected.json b/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module_chdir/expected.json index 07d230b50f2..8168a399907 100644 --- a/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module_chdir/expected.json +++ b/internal/providers/terraform/testdata/hcl_provider_test/adds_source_url_from_remote_module_chdir/expected.json @@ -254,15 +254,15 @@ { "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_acl.this", - "startLine": 65, - "endLine": 102 + "startLine": 66, + "endLine": 103 } ], "checksum": "2a79b34cee7ece7b7a5231adf7ebbf8efc9b246e0b02c43ff0857bb70742ac53", - "endLine": 102, + "endLine": 103, "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 65 + "startLine": 66 } }, { @@ -291,15 +291,15 @@ { "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 922, - "endLine": 937 + "startLine": 923, + "endLine": 938 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 937, + "endLine": 938, "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 922 + "startLine": 923 } }, { @@ -327,15 +327,15 @@ { "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 911, - "endLine": 920 + "startLine": 912, + "endLine": 921 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 920, + "endLine": 921, "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 911 + "startLine": 912 } }, { @@ -367,15 +367,15 @@ { "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_versioning.this", - "startLine": 159, - "endLine": 173 + "startLine": 160, + "endLine": 174 } ], "checksum": "d87dc101b6296498b9a2114efae840b7b1b1173c2e10fa9b312ae39df0142db5", - "endLine": 173, + "endLine": 174, "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 159 + "startLine": 160 } } ], @@ -1078,15 +1078,15 @@ { "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_acl.this", - "startLine": 65, - "endLine": 102 + "startLine": 66, + "endLine": 103 } ], "checksum": "2a79b34cee7ece7b7a5231adf7ebbf8efc9b246e0b02c43ff0857bb70742ac53", - "endLine": 102, + "endLine": 103, "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 65 + "startLine": 66 } }, { @@ -1115,15 +1115,15 @@ { "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_ownership_controls.this", - "startLine": 922, - "endLine": 937 + "startLine": 923, + "endLine": 938 } ], "checksum": "30169ea40fd2d6891c592ed40354fe60e9dd50d530309030802fafc49eb3f9bb", - "endLine": 937, + "endLine": 938, "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 922 + "startLine": 923 } }, { @@ -1151,15 +1151,15 @@ { "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_public_access_block.this", - "startLine": 911, - "endLine": 920 + "startLine": 912, + "endLine": 921 } ], "checksum": "37c21c291bc3f8b8e8594a769ba27d4c164473569562b3f4fc7661e58d8e6917", - "endLine": 920, + "endLine": 921, "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 911 + "startLine": 912 } }, { @@ -1191,15 +1191,15 @@ { "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "blockName": "aws_s3_bucket_versioning.this", - "startLine": 159, - "endLine": 173 + "startLine": 160, + "endLine": 174 } ], "checksum": "d87dc101b6296498b9a2114efae840b7b1b1173c2e10fa9b312ae39df0142db5", - "endLine": 173, + "endLine": 174, "filename": ".infracost/terraform_modules/2d1f98832c35b580d7d2ddad6b43dc8a/main.tf", "moduleFilename": "https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/blob/v4.1.1/main.tf", - "startLine": 159 + "startLine": 160 } } ], From 3662d574db93ab1f745490fbac0722771b3319d1 Mon Sep 17 00:00:00 2001 From: Hugo Rut Date: Tue, 24 Dec 2024 12:23:09 +0000 Subject: [PATCH 74/81] feat: add treeFile output for generate command (#3291) Adds the ability to save the infracost autodetect logic to a file. The format is saved as a simplified tree structure which highlights the root projects and env files that have been discovered in the project/repository. --- cmd/infracost/generate.go | 15 +++ cmd/infracost/generate_test.go | 38 ++++++- .../completion_shell_bash.golden | 4 + cmd/infracost/testdata/generate/.gitignore | 2 + .../generate/aunt/expected-tree.golden | 11 ++ .../aunt_and_great_aunt/expected-tree.golden | 16 +++ .../expected-tree.golden | 14 +++ .../expected-tree.golden | 23 ++++ .../generate/child/expected-tree.golden | 11 ++ .../child_with_dot/expected-tree.golden | 6 ++ .../generate/cousin/expected-tree.golden | 19 ++++ .../generate/cousin_flat/expected-tree.golden | 8 ++ .../detected_projects/expected-tree.golden | 6 ++ .../expected-tree.golden | 6 ++ .../generate/env_dirs/expected-tree.golden | 10 ++ .../generate/env_names/expected-tree.golden | 10 ++ .../expected-tree.golden | 10 ++ .../env_names_in_path/expected-tree.golden | 12 +++ .../env_names_partials/expected-tree.golden | 8 ++ .../expected-tree.golden | 8 ++ .../env_var_dots/expected-tree.golden | 10 ++ .../env_var_extensions/expected-tree.golden | 9 ++ .../excluded_dirs/expected-tree.golden | 5 + .../excluded_dirs_glob/expected-tree.golden | 3 + .../excludes_examples/expected-tree.golden | 1 + .../external_tfvars/expected-tree.golden | 12 +++ .../force_project_type/expected-tree.golden | 4 + .../generate/grandchild/expected-tree.golden | 13 +++ .../generate/grandparent/expected-tree.golden | 10 ++ .../generate/great_aunt/expected-tree.golden | 12 +++ .../include_dirs/expected-tree.golden | 12 +++ .../includes_hidden_dirs/expected-tree.golden | 3 + .../max_search_depth/expected-tree.golden | 2 + .../module_calls/expected-tree.golden | 5 + .../expected-tree.golden | 5 + .../expected-tree.golden | 8 ++ .../multi_descendents/expected-tree.golden | 19 ++++ .../generate/parent/expected-tree.golden | 6 ++ .../expected-tree.golden | 16 +++ .../expected-tree.golden | 10 ++ .../path_overrides/expected-tree.golden | 12 +++ .../prefer_folder_name/expected-tree.golden | 6 ++ .../generate/repo_name/expected-tree.golden | 7 ++ .../generate/root_paths/expected-tree.golden | 7 ++ .../root_with_leafs/expected-tree.golden | 11 ++ .../shared_env_var_files/expected-tree.golden | 9 ++ .../generate/sibling/expected-tree.golden | 7 ++ .../sibling_and_aunt/expected-tree.golden | 19 ++++ .../expected-tree.golden | 6 ++ .../expected-tree.golden | 10 ++ .../expected-tree.golden | 9 ++ .../expected-tree.golden | 2 + .../single_project/expected-tree.golden | 3 + .../single_root_project/expected-tree.golden | 1 + .../generate/terragrunt/expected-tree.golden | 5 + .../expected-tree.golden | 9 ++ .../generate/tfvar_json/expected-tree.golden | 7 ++ .../wildcard_env_names/expected-tree.golden | 11 ++ .../expected-tree.golden | 4 + examples/terraform/modules/foo/main.tf | 26 +++++ internal/hcl/project_locator.go | 101 +++++++++++++++++- internal/providers/detect.go | 9 +- .../providers/terraform/hcl_provider_test.go | 2 +- 63 files changed, 677 insertions(+), 8 deletions(-) create mode 100755 cmd/infracost/testdata/generate/aunt/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/aunt_and_great_aunt/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/aunt_filename_matches_project/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/aunt_filenames_with_env/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/child/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/child_with_dot/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/cousin/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/cousin_flat/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/detected_projects/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/detected_root_modules/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/env_dirs/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/env_names/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/env_names_case_insensitive/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/env_names_in_path/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/env_names_partials/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/env_var_dot_extensions/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/env_var_dots/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/env_var_extensions/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/excluded_dirs/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/excluded_dirs_glob/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/excludes_examples/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/external_tfvars/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/force_project_type/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/grandchild/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/grandparent/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/great_aunt/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/include_dirs/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/includes_hidden_dirs/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/max_search_depth/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/module_calls/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/module_calls_with_template/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/modules_and_external_tfvars/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/multi_descendents/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/parent/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/parent_and_grandparent/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/parent_filename_matches_project/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/path_overrides/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/prefer_folder_name/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/repo_name/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/root_paths/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/root_with_leafs/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/shared_env_var_files/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/sibling/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/sibling_and_aunt/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/sibling_and_child_default/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/sibling_and_child_prefer_child/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/sibling_and_child_prefer_sibling/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/single_nested_project/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/single_project/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/single_root_project/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/terragrunt/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/terragrunt_and_terraform_mixed/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/tfvar_json/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/wildcard_env_names/expected-tree.golden create mode 100755 cmd/infracost/testdata/generate/with_no_directories_with_providers/expected-tree.golden create mode 100644 examples/terraform/modules/foo/main.tf diff --git a/cmd/infracost/generate.go b/cmd/infracost/generate.go index 1d1b4c46dce..3c44b98d973 100644 --- a/cmd/infracost/generate.go +++ b/cmd/infracost/generate.go @@ -26,6 +26,7 @@ type generateConfigCommand struct { templatePath string template string outFile string + treeFile string } func newGenerateConfigCommand() *cobra.Command { @@ -48,6 +49,7 @@ func newGenerateConfigCommand() *cobra.Command { cmd.Flags().StringVar(&gen.template, "template", "", "Infracost template string that will generate the config-file yaml output") cmd.Flags().StringVar(&gen.templatePath, "template-path", "", "Path to the Infracost template file that will generate the config-file yaml output") cmd.Flags().StringVar(&gen.outFile, "out-file", "", "Save output to a file") + cmd.Flags().StringVar(&gen.treeFile, "tree-file", "", "Save a simplified tree of the detected projects and var files to a file") return cmd } @@ -103,6 +105,19 @@ func (g *generateConfigCommand) run(cmd *cobra.Command, args []string) error { } } + if g.treeFile != "" { + treeFile, err := os.Create(g.treeFile) + if err != nil { + logging.Logger.Warn().Msgf("could not create detected tree file: at %s %s", g.treeFile, err) + } else { + _, err = treeFile.WriteString(detectionOutput.Tree) + if err != nil { + logging.Logger.Warn().Msgf("could not write detected tree file: %s", err) + } + _ = treeFile.Close() + } + } + for _, provider := range detectionOutput.Providers { if v, ok := provider.(hcl.DetectedProject); ok { autoProjects = append(autoProjects, v) diff --git a/cmd/infracost/generate_test.go b/cmd/infracost/generate_test.go index 26aa5ad4bd7..51e5ac16be2 100644 --- a/cmd/infracost/generate_test.go +++ b/cmd/infracost/generate_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/infracost/infracost/internal/config" "github.com/infracost/infracost/internal/logging" @@ -109,13 +110,48 @@ func TestGenerateConfigAutoDetect(t *testing.T) { }) return + } else { + assertGoldenFile(tt, args, dir) } + }) + + t.Run(name+"-tree", func(tt *testing.T) { + tempDir := tt.TempDir() + testutil.CreateDirectoryStructure(tt, path.Join(dir, "tree.txt"), tempDir) - assertGoldenFile(tt, args, dir) + args := []string{ + "generate", + "config", + "--repo-path", + tempDir, + "--tree-file", + path.Join(dir, "infracost-tree.txt"), + } + templatePath := path.Join(dir, "infracost.yml.tmpl") + if _, err := os.Stat(templatePath); err == nil { + args = append(args, "--template-path", templatePath) + } + + assertGoldenFileTree(tt, args, dir) }) } } +func assertGoldenFileTree(tt *testing.T, args []string, dir string) { + GetCommandOutput(tt, args, nil) + + actual, err := os.ReadFile(path.Join(dir, "infracost-tree.txt")) + require.NoError(tt, err) + + if _, err := os.Stat(path.Join(dir, "expected-tree.golden")); err != nil { + err := os.WriteFile(path.Join(dir, "expected-tree.golden"), actual, 0600) + assert.NoError(tt, err) + return + } + + testutil.AssertGoldenFile(tt, path.Join(dir, "expected-tree.golden"), actual) +} + func assertGoldenFile(tt *testing.T, args []string, dir string) { buf := bytes.NewBuffer([]byte{}) actual := GetCommandOutput(tt, args, nil, func(ctx *config.RunContext) { diff --git a/cmd/infracost/testdata/completion_shell_bash/completion_shell_bash.golden b/cmd/infracost/testdata/completion_shell_bash/completion_shell_bash.golden index 296ebf5d81f..e1180e65442 100644 --- a/cmd/infracost/testdata/completion_shell_bash/completion_shell_bash.golden +++ b/cmd/infracost/testdata/completion_shell_bash/completion_shell_bash.golden @@ -1104,6 +1104,10 @@ _infracost_generate_config() two_word_flags+=("--template-path") local_nonpersistent_flags+=("--template-path") local_nonpersistent_flags+=("--template-path=") + flags+=("--tree-file=") + two_word_flags+=("--tree-file") + local_nonpersistent_flags+=("--tree-file") + local_nonpersistent_flags+=("--tree-file=") flags+=("--debug-report") flags+=("--log-level=") two_word_flags+=("--log-level") diff --git a/cmd/infracost/testdata/generate/.gitignore b/cmd/infracost/testdata/generate/.gitignore index d2dc29bb01e..f64d9f5b7d6 100644 --- a/cmd/infracost/testdata/generate/.gitignore +++ b/cmd/infracost/testdata/generate/.gitignore @@ -1 +1,3 @@ actual.txt +actual-tree.txt +infracost-tree.txt \ No newline at end of file diff --git a/cmd/infracost/testdata/generate/aunt/expected-tree.golden b/cmd/infracost/testdata/generate/aunt/expected-tree.golden new file mode 100755 index 00000000000..52a79af2cc7 --- /dev/null +++ b/cmd/infracost/testdata/generate/aunt/expected-tree.golden @@ -0,0 +1,11 @@ +└── apps + ├── bar + │ ├── us1 (terraform) + │ └── us2 (terraform) + ├── envs + │ ├── default.tfvars + │ ├── dev.tfvars + │ └── prod.tfvars + └── foo + ├── us1 (terraform) + └── us2 (terraform) diff --git a/cmd/infracost/testdata/generate/aunt_and_great_aunt/expected-tree.golden b/cmd/infracost/testdata/generate/aunt_and_great_aunt/expected-tree.golden new file mode 100755 index 00000000000..edce97d9528 --- /dev/null +++ b/cmd/infracost/testdata/generate/aunt_and_great_aunt/expected-tree.golden @@ -0,0 +1,16 @@ +└── infra + ├── apps + │ ├── bar + │ │ ├── us1 (terraform) + │ │ └── us2 (terraform) + │ ├── envs + │ │ ├── default.tfvars + │ │ ├── dev.tfvars + │ │ └── prod.tfvars + │ └── foo + │ ├── us1 (terraform) + │ └── us2 (terraform) + └── envs + ├── default.tfvars + ├── dev.tfvars + └── prod.tfvars diff --git a/cmd/infracost/testdata/generate/aunt_filename_matches_project/expected-tree.golden b/cmd/infracost/testdata/generate/aunt_filename_matches_project/expected-tree.golden new file mode 100755 index 00000000000..b931d71397e --- /dev/null +++ b/cmd/infracost/testdata/generate/aunt_filename_matches_project/expected-tree.golden @@ -0,0 +1,14 @@ +└── infra + ├── components + │ ├── bar (terraform) + │ ├── baz (terraform) + │ └── foo (terraform) + └── variables + └── env + ├── dev + │ ├── bar.tfvars + │ └── defaults.tfvars + └── prod + ├── bar.tfvars + ├── defaults.tfvars + └── foo.tfvars diff --git a/cmd/infracost/testdata/generate/aunt_filenames_with_env/expected-tree.golden b/cmd/infracost/testdata/generate/aunt_filenames_with_env/expected-tree.golden new file mode 100755 index 00000000000..06d2c354263 --- /dev/null +++ b/cmd/infracost/testdata/generate/aunt_filenames_with_env/expected-tree.golden @@ -0,0 +1,23 @@ +├── components +│ ├── age (terraform) +│ ├── airflow (terraform) +│ └── apm-events (terraform) +└── variables + └── default.tfvars + ├── cko-dev + │ ├── age.tfvars + │ ├── airflow.tfvars + │ ├── apm-events.tfvars + │ └── dev.tfvars + ├── cko-mgmt + │ ├── age.tfvars + │ ├── airflow.tfvars + │ └── apm-events.tfvars + ├── cko-playground + │ ├── age.tfvars + │ ├── airflow.tfvars + │ └── apm-events.tfvars + └── cko-prod + ├── age.tfvars + ├── airflow.tfvars + └── apm-events.tfvars diff --git a/cmd/infracost/testdata/generate/child/expected-tree.golden b/cmd/infracost/testdata/generate/child/expected-tree.golden new file mode 100755 index 00000000000..12c8fbdba5d --- /dev/null +++ b/cmd/infracost/testdata/generate/child/expected-tree.golden @@ -0,0 +1,11 @@ +└── apps + ├── bar (terraform) + │ └── envs + │ ├── default.tfvars + │ ├── dev.tfvars + │ └── prod.tfvars + └── foo (terraform) + └── envs + ├── default.tfvars + ├── dev.tfvars + └── staging.tfvars diff --git a/cmd/infracost/testdata/generate/child_with_dot/expected-tree.golden b/cmd/infracost/testdata/generate/child_with_dot/expected-tree.golden new file mode 100755 index 00000000000..a8247ee7ef3 --- /dev/null +++ b/cmd/infracost/testdata/generate/child_with_dot/expected-tree.golden @@ -0,0 +1,6 @@ +└── apps + └── foo (terraform) + └── envs + ├── default.tfvars + ├── dev.eu-west-1.tfvars + └── staging.eu-west-1.tfvars diff --git a/cmd/infracost/testdata/generate/cousin/expected-tree.golden b/cmd/infracost/testdata/generate/cousin/expected-tree.golden new file mode 100755 index 00000000000..8a668a4a514 --- /dev/null +++ b/cmd/infracost/testdata/generate/cousin/expected-tree.golden @@ -0,0 +1,19 @@ +└── infra + ├── components + │ └── foo (terraform) + ├── nested + │ ├── components + │ │ └── baz (terraform) + │ └── variables + │ └── envs + │ └── defaults.tfvars + │ ├── dev + │ │ └── dev.tfvars + │ └── stag + │ └── stag.tfvars + └── variables + └── envs + ├── dev + │ └── dev.tfvars + └── prod + └── prod.tfvars diff --git a/cmd/infracost/testdata/generate/cousin_flat/expected-tree.golden b/cmd/infracost/testdata/generate/cousin_flat/expected-tree.golden new file mode 100755 index 00000000000..34f75cb651a --- /dev/null +++ b/cmd/infracost/testdata/generate/cousin_flat/expected-tree.golden @@ -0,0 +1,8 @@ +├── . (terraform) +└── variables + └── envs + └── defaults.tfvars + ├── dev + │ └── dev.tfvars + └── prod + └── prod.tfvars diff --git a/cmd/infracost/testdata/generate/detected_projects/expected-tree.golden b/cmd/infracost/testdata/generate/detected_projects/expected-tree.golden new file mode 100755 index 00000000000..721e5e688d4 --- /dev/null +++ b/cmd/infracost/testdata/generate/detected_projects/expected-tree.golden @@ -0,0 +1,6 @@ +└── apps + ├── default.tfvars + ├── dev.tfvars + └── prod.tfvars + ├── bar (terraform) + └── foo (terraform) diff --git a/cmd/infracost/testdata/generate/detected_root_modules/expected-tree.golden b/cmd/infracost/testdata/generate/detected_root_modules/expected-tree.golden new file mode 100755 index 00000000000..721e5e688d4 --- /dev/null +++ b/cmd/infracost/testdata/generate/detected_root_modules/expected-tree.golden @@ -0,0 +1,6 @@ +└── apps + ├── default.tfvars + ├── dev.tfvars + └── prod.tfvars + ├── bar (terraform) + └── foo (terraform) diff --git a/cmd/infracost/testdata/generate/env_dirs/expected-tree.golden b/cmd/infracost/testdata/generate/env_dirs/expected-tree.golden new file mode 100755 index 00000000000..5e19be453d9 --- /dev/null +++ b/cmd/infracost/testdata/generate/env_dirs/expected-tree.golden @@ -0,0 +1,10 @@ +└── infra + ├── components + │ ├── baz (terraform) + │ └── foo (terraform) + └── variables + └── defaults.tfvars + ├── dev + │ └── bla.tfvars + └── stag + └── bop.tfvars diff --git a/cmd/infracost/testdata/generate/env_names/expected-tree.golden b/cmd/infracost/testdata/generate/env_names/expected-tree.golden new file mode 100755 index 00000000000..a641330e473 --- /dev/null +++ b/cmd/infracost/testdata/generate/env_names/expected-tree.golden @@ -0,0 +1,10 @@ +└── apps + ├── bat.tfvars + ├── baz.tfvars + ├── default.tfvars + ├── dev.tfvars + ├── network-bat.tfvars + ├── network-baz.tfvars + └── prod.tfvars + ├── bar (terraform) + └── foo (terraform) diff --git a/cmd/infracost/testdata/generate/env_names_case_insensitive/expected-tree.golden b/cmd/infracost/testdata/generate/env_names_case_insensitive/expected-tree.golden new file mode 100755 index 00000000000..415d58a5414 --- /dev/null +++ b/cmd/infracost/testdata/generate/env_names_case_insensitive/expected-tree.golden @@ -0,0 +1,10 @@ +└── apps + ├── bat.tfvars + ├── baz.tfvars + ├── default.tfvars + ├── dev.tfvars + ├── network-Bat.tfvars + ├── network-Baz.tfvars + └── prod.tfvars + ├── bar (terraform) + └── foo (terraform) diff --git a/cmd/infracost/testdata/generate/env_names_in_path/expected-tree.golden b/cmd/infracost/testdata/generate/env_names_in_path/expected-tree.golden new file mode 100755 index 00000000000..a785aaf7308 --- /dev/null +++ b/cmd/infracost/testdata/generate/env_names_in_path/expected-tree.golden @@ -0,0 +1,12 @@ +└── infra + ├── components + │ ├── foo-dev (terraform) + │ └── foo-prod (terraform) + └── variables + └── defaults.tfvars + ├── dev + │ └── foo-dev.tfvars + ├── prod + │ └── foo-prod.tfvars + └── qa + └── bar.tfvars diff --git a/cmd/infracost/testdata/generate/env_names_partials/expected-tree.golden b/cmd/infracost/testdata/generate/env_names_partials/expected-tree.golden new file mode 100755 index 00000000000..ae170770f33 --- /dev/null +++ b/cmd/infracost/testdata/generate/env_names_partials/expected-tree.golden @@ -0,0 +1,8 @@ +└── apps + ├── default.tfvars + ├── dev.tfvars + ├── network-bat.tfvars + ├── network-baz.tfvars + └── prod.tfvars + ├── bar (terraform) + └── foo (terraform) diff --git a/cmd/infracost/testdata/generate/env_var_dot_extensions/expected-tree.golden b/cmd/infracost/testdata/generate/env_var_dot_extensions/expected-tree.golden new file mode 100755 index 00000000000..73ae26bc36c --- /dev/null +++ b/cmd/infracost/testdata/generate/env_var_dot_extensions/expected-tree.golden @@ -0,0 +1,8 @@ +└── apps + ├── .config.dev.env.tfvars + ├── .config.prod.env.tfvars + ├── .dev-custom-ext + ├── .prod-custom-ext + └── default.tfvars + ├── bar (terraform) + └── foo (terraform) diff --git a/cmd/infracost/testdata/generate/env_var_dots/expected-tree.golden b/cmd/infracost/testdata/generate/env_var_dots/expected-tree.golden new file mode 100755 index 00000000000..9a17da6100b --- /dev/null +++ b/cmd/infracost/testdata/generate/env_var_dots/expected-tree.golden @@ -0,0 +1,10 @@ +└── apps + ├── .dev.tfvars + ├── .network-bat.tfvars + ├── .network-baz.tfvars + ├── .prod.tfvars + ├── config.dev.tfvars + ├── config.prod.tfvars + └── default.tfvars + ├── bar (terraform) + └── foo (terraform) diff --git a/cmd/infracost/testdata/generate/env_var_extensions/expected-tree.golden b/cmd/infracost/testdata/generate/env_var_extensions/expected-tree.golden new file mode 100755 index 00000000000..c1cf9ae60bb --- /dev/null +++ b/cmd/infracost/testdata/generate/env_var_extensions/expected-tree.golden @@ -0,0 +1,9 @@ +└── apps + ├── baz-custom-ext + ├── common.tfvars.json + ├── default.tfvars + ├── dev.tfvars + ├── prod-custom-ext + └── prod.env.tfvars + ├── bar (terraform) + └── foo (terraform) diff --git a/cmd/infracost/testdata/generate/excluded_dirs/expected-tree.golden b/cmd/infracost/testdata/generate/excluded_dirs/expected-tree.golden new file mode 100755 index 00000000000..d5f50f24477 --- /dev/null +++ b/cmd/infracost/testdata/generate/excluded_dirs/expected-tree.golden @@ -0,0 +1,5 @@ +└── apps + ├── default.tfvars + ├── dev.tfvars + └── prod.tfvars + └── bar (terraform) diff --git a/cmd/infracost/testdata/generate/excluded_dirs_glob/expected-tree.golden b/cmd/infracost/testdata/generate/excluded_dirs_glob/expected-tree.golden new file mode 100755 index 00000000000..f2acc734dfe --- /dev/null +++ b/cmd/infracost/testdata/generate/excluded_dirs_glob/expected-tree.golden @@ -0,0 +1,3 @@ +└── apps + ├── bar (terraform) + └── foo (terraform) diff --git a/cmd/infracost/testdata/generate/excludes_examples/expected-tree.golden b/cmd/infracost/testdata/generate/excludes_examples/expected-tree.golden new file mode 100755 index 00000000000..8393e1f359e --- /dev/null +++ b/cmd/infracost/testdata/generate/excludes_examples/expected-tree.golden @@ -0,0 +1 @@ +└── infra (terraform) diff --git a/cmd/infracost/testdata/generate/external_tfvars/expected-tree.golden b/cmd/infracost/testdata/generate/external_tfvars/expected-tree.golden new file mode 100755 index 00000000000..6456e664d78 --- /dev/null +++ b/cmd/infracost/testdata/generate/external_tfvars/expected-tree.golden @@ -0,0 +1,12 @@ +├── apps +│ └── foo (terraform) +├── envs +│ └── default.tfvars +│ ├── dev +│ │ └── dev.tfvars +│ ├── prod +│ │ └── prod.tfvars +│ └── test +│ └── test.tfvars +└── infra + └── db (terraform) diff --git a/cmd/infracost/testdata/generate/force_project_type/expected-tree.golden b/cmd/infracost/testdata/generate/force_project_type/expected-tree.golden new file mode 100755 index 00000000000..1f41a2ef569 --- /dev/null +++ b/cmd/infracost/testdata/generate/force_project_type/expected-tree.golden @@ -0,0 +1,4 @@ +├── dup (terraform) +│ ├── dev.tfvars +│ └── prod.tfvars +└── nondup (terragrunt) diff --git a/cmd/infracost/testdata/generate/grandchild/expected-tree.golden b/cmd/infracost/testdata/generate/grandchild/expected-tree.golden new file mode 100755 index 00000000000..1b69798e26e --- /dev/null +++ b/cmd/infracost/testdata/generate/grandchild/expected-tree.golden @@ -0,0 +1,13 @@ +└── apps + ├── bar (terraform) + │ └── config + │ └── envs + │ ├── default.tfvars + │ ├── dev.tfvars + │ └── prod.tfvars + └── foo (terraform) + └── config + └── envs + ├── default.tfvars + ├── dev.tfvars + └── staging.tfvars diff --git a/cmd/infracost/testdata/generate/grandparent/expected-tree.golden b/cmd/infracost/testdata/generate/grandparent/expected-tree.golden new file mode 100755 index 00000000000..e0febd3d531 --- /dev/null +++ b/cmd/infracost/testdata/generate/grandparent/expected-tree.golden @@ -0,0 +1,10 @@ +└── apps + ├── default.tfvars + ├── dev.tfvars + └── prod.tfvars + ├── bar + │ ├── us1 (terraform) + │ └── us2 (terraform) + └── foo + ├── us1 (terraform) + └── us2 (terraform) diff --git a/cmd/infracost/testdata/generate/great_aunt/expected-tree.golden b/cmd/infracost/testdata/generate/great_aunt/expected-tree.golden new file mode 100755 index 00000000000..b64b366b3c9 --- /dev/null +++ b/cmd/infracost/testdata/generate/great_aunt/expected-tree.golden @@ -0,0 +1,12 @@ +└── infra + ├── apps + │ ├── bar + │ │ ├── us1 (terraform) + │ │ └── us2 (terraform) + │ └── foo + │ ├── us1 (terraform) + │ └── us2 (terraform) + └── envs + ├── default.tfvars + ├── dev.tfvars + └── prod.tfvars diff --git a/cmd/infracost/testdata/generate/include_dirs/expected-tree.golden b/cmd/infracost/testdata/generate/include_dirs/expected-tree.golden new file mode 100755 index 00000000000..ea9f80a05eb --- /dev/null +++ b/cmd/infracost/testdata/generate/include_dirs/expected-tree.golden @@ -0,0 +1,12 @@ +└── apps + ├── default.tfvars + ├── dev.tfvars + └── prod.tfvars + ├── .hidden (terraform) + ├── bar (terraform) + ├── bat (terraform) + ├── baz (terraform) + ├── foo (terraform) + └── wildcard + ├── one (terraform) + └── two (terraform) diff --git a/cmd/infracost/testdata/generate/includes_hidden_dirs/expected-tree.golden b/cmd/infracost/testdata/generate/includes_hidden_dirs/expected-tree.golden new file mode 100755 index 00000000000..696ab579bb0 --- /dev/null +++ b/cmd/infracost/testdata/generate/includes_hidden_dirs/expected-tree.golden @@ -0,0 +1,3 @@ +└── .infra + ├── bat (terraform) + └── baz (terraform) diff --git a/cmd/infracost/testdata/generate/max_search_depth/expected-tree.golden b/cmd/infracost/testdata/generate/max_search_depth/expected-tree.golden new file mode 100755 index 00000000000..61fc279d67e --- /dev/null +++ b/cmd/infracost/testdata/generate/max_search_depth/expected-tree.golden @@ -0,0 +1,2 @@ +└── infra + └── foo (terraform) diff --git a/cmd/infracost/testdata/generate/module_calls/expected-tree.golden b/cmd/infracost/testdata/generate/module_calls/expected-tree.golden new file mode 100755 index 00000000000..8da37972b33 --- /dev/null +++ b/cmd/infracost/testdata/generate/module_calls/expected-tree.golden @@ -0,0 +1,5 @@ +└── infra + ├── dev (terraform) + ├── modules + │ └── is_a_project (terraform) + └── prod (terraform) diff --git a/cmd/infracost/testdata/generate/module_calls_with_template/expected-tree.golden b/cmd/infracost/testdata/generate/module_calls_with_template/expected-tree.golden new file mode 100755 index 00000000000..8da37972b33 --- /dev/null +++ b/cmd/infracost/testdata/generate/module_calls_with_template/expected-tree.golden @@ -0,0 +1,5 @@ +└── infra + ├── dev (terraform) + ├── modules + │ └── is_a_project (terraform) + └── prod (terraform) diff --git a/cmd/infracost/testdata/generate/modules_and_external_tfvars/expected-tree.golden b/cmd/infracost/testdata/generate/modules_and_external_tfvars/expected-tree.golden new file mode 100755 index 00000000000..b0fed3a4617 --- /dev/null +++ b/cmd/infracost/testdata/generate/modules_and_external_tfvars/expected-tree.golden @@ -0,0 +1,8 @@ +├── . +│ └── default.tfvars +├── db +│ ├── dev (terraform) +│ └── prod (terraform) +└── envs + ├── dev.tfvars + └── prod.tfvars diff --git a/cmd/infracost/testdata/generate/multi_descendents/expected-tree.golden b/cmd/infracost/testdata/generate/multi_descendents/expected-tree.golden new file mode 100755 index 00000000000..3a73853c063 --- /dev/null +++ b/cmd/infracost/testdata/generate/multi_descendents/expected-tree.golden @@ -0,0 +1,19 @@ +└── apps + ├── bar (terraform) + │ ├── envs + │ │ ├── dev.tfvars + │ │ └── staging.tfvars + │ └── us1 (terraform) + │ └── envs + │ ├── dev.tfvars + │ └── prod.tfvars + ├── envs + │ └── default.tfvars + └── foo (terraform) + ├── envs + │ ├── dev.tfvars + │ └── staging.tfvars + └── us1 (terraform) + └── envs + ├── dev.tfvars + └── prod.tfvars diff --git a/cmd/infracost/testdata/generate/parent/expected-tree.golden b/cmd/infracost/testdata/generate/parent/expected-tree.golden new file mode 100755 index 00000000000..721e5e688d4 --- /dev/null +++ b/cmd/infracost/testdata/generate/parent/expected-tree.golden @@ -0,0 +1,6 @@ +└── apps + ├── default.tfvars + ├── dev.tfvars + └── prod.tfvars + ├── bar (terraform) + └── foo (terraform) diff --git a/cmd/infracost/testdata/generate/parent_and_grandparent/expected-tree.golden b/cmd/infracost/testdata/generate/parent_and_grandparent/expected-tree.golden new file mode 100755 index 00000000000..935943eb14c --- /dev/null +++ b/cmd/infracost/testdata/generate/parent_and_grandparent/expected-tree.golden @@ -0,0 +1,16 @@ +└── apps + ├── default.tfvars + ├── dev.tfvars + └── prod.tfvars + ├── bar + │ ├── default.tfvars + │ ├── dev.tfvars + │ └── prod.tfvars + │ ├── us1 (terraform) + │ └── us2 (terraform) + └── foo + ├── default.tfvars + ├── dev.tfvars + └── prod.tfvars + ├── us1 (terraform) + └── us2 (terraform) diff --git a/cmd/infracost/testdata/generate/parent_filename_matches_project/expected-tree.golden b/cmd/infracost/testdata/generate/parent_filename_matches_project/expected-tree.golden new file mode 100755 index 00000000000..39a2b01cb6a --- /dev/null +++ b/cmd/infracost/testdata/generate/parent_filename_matches_project/expected-tree.golden @@ -0,0 +1,10 @@ +└── infra + ├── baz.tfvars + └── foo.tfvars + └── components + ├── bar.tfvars + ├── defaults.tfvars + └── foo.tfvars + ├── bar (terraform) + ├── baz (terraform) + └── foo (terraform) diff --git a/cmd/infracost/testdata/generate/path_overrides/expected-tree.golden b/cmd/infracost/testdata/generate/path_overrides/expected-tree.golden new file mode 100755 index 00000000000..6c403a9bcff --- /dev/null +++ b/cmd/infracost/testdata/generate/path_overrides/expected-tree.golden @@ -0,0 +1,12 @@ +└── infra + ├── bat.tfvars + ├── baz.tfvars + ├── bip.tfvars + ├── default.tfvars + ├── network-bat.tfvars + └── network-baz.tfvars + └── components + ├── bar (terraform) + ├── blah (terraform) + └── foo (terraform) + └── var.auto.tfvars diff --git a/cmd/infracost/testdata/generate/prefer_folder_name/expected-tree.golden b/cmd/infracost/testdata/generate/prefer_folder_name/expected-tree.golden new file mode 100755 index 00000000000..4ad5ff3761e --- /dev/null +++ b/cmd/infracost/testdata/generate/prefer_folder_name/expected-tree.golden @@ -0,0 +1,6 @@ +└── infra (terraform) + └── envs + ├── qa + │ └── prod.tfvars + └── stg + └── dev.tfvars diff --git a/cmd/infracost/testdata/generate/repo_name/expected-tree.golden b/cmd/infracost/testdata/generate/repo_name/expected-tree.golden new file mode 100755 index 00000000000..c29765f26f0 --- /dev/null +++ b/cmd/infracost/testdata/generate/repo_name/expected-tree.golden @@ -0,0 +1,7 @@ +└── apps + ├── bar (terraform) + │ └── terraform.tfvars + └── foo (terraform) + ├── dev.tfvars + ├── prod.tfvars + └── terraform.tfvars diff --git a/cmd/infracost/testdata/generate/root_paths/expected-tree.golden b/cmd/infracost/testdata/generate/root_paths/expected-tree.golden new file mode 100755 index 00000000000..c29765f26f0 --- /dev/null +++ b/cmd/infracost/testdata/generate/root_paths/expected-tree.golden @@ -0,0 +1,7 @@ +└── apps + ├── bar (terraform) + │ └── terraform.tfvars + └── foo (terraform) + ├── dev.tfvars + ├── prod.tfvars + └── terraform.tfvars diff --git a/cmd/infracost/testdata/generate/root_with_leafs/expected-tree.golden b/cmd/infracost/testdata/generate/root_with_leafs/expected-tree.golden new file mode 100755 index 00000000000..6fa0d2404e3 --- /dev/null +++ b/cmd/infracost/testdata/generate/root_with_leafs/expected-tree.golden @@ -0,0 +1,11 @@ +└── terraform (terraform) + ├── config + │ ├── dev + │ │ └── terraform.tfvars + │ └── prod + │ └── terraform.tfvars + └── foo (terraform) + ├── dev + │ └── terraform.tfvars + └── prod + └── terraform.tfvars diff --git a/cmd/infracost/testdata/generate/shared_env_var_files/expected-tree.golden b/cmd/infracost/testdata/generate/shared_env_var_files/expected-tree.golden new file mode 100755 index 00000000000..272a9fc55d8 --- /dev/null +++ b/cmd/infracost/testdata/generate/shared_env_var_files/expected-tree.golden @@ -0,0 +1,9 @@ +└── apps + ├── default.tfvars + ├── dev-default.tfvars + ├── prod-default.tfvars + └── staging-default.tfvars + ├── bar (terraform) + │ ├── dev.tfvars + │ └── staging.tfvars + └── foo (terraform) diff --git a/cmd/infracost/testdata/generate/sibling/expected-tree.golden b/cmd/infracost/testdata/generate/sibling/expected-tree.golden new file mode 100755 index 00000000000..dc25de4e570 --- /dev/null +++ b/cmd/infracost/testdata/generate/sibling/expected-tree.golden @@ -0,0 +1,7 @@ +└── apps + ├── bar (terraform) + ├── envs + │ ├── dev.tfvars + │ ├── prod.tfvars + │ └── shared.tfvars + └── foo (terraform) diff --git a/cmd/infracost/testdata/generate/sibling_and_aunt/expected-tree.golden b/cmd/infracost/testdata/generate/sibling_and_aunt/expected-tree.golden new file mode 100755 index 00000000000..99671657d16 --- /dev/null +++ b/cmd/infracost/testdata/generate/sibling_and_aunt/expected-tree.golden @@ -0,0 +1,19 @@ +└── apps + ├── bar + │ ├── envs + │ │ ├── default.tfvars + │ │ ├── dev.tfvars + │ │ └── prod.tfvars + │ ├── us1 (terraform) + │ └── us2 (terraform) + ├── envs + │ ├── default.tfvars + │ ├── dev.tfvars + │ └── prod.tfvars + └── foo + ├── envs + │ ├── default.tfvars + │ ├── dev.tfvars + │ └── prod.tfvars + ├── us1 (terraform) + └── us2 (terraform) diff --git a/cmd/infracost/testdata/generate/sibling_and_child_default/expected-tree.golden b/cmd/infracost/testdata/generate/sibling_and_child_default/expected-tree.golden new file mode 100755 index 00000000000..29ff4aa2ca2 --- /dev/null +++ b/cmd/infracost/testdata/generate/sibling_and_child_default/expected-tree.golden @@ -0,0 +1,6 @@ +└── apps (terraform) + ├── bar (terraform) + ├── envs + │ ├── dev.tfvars + │ └── prod.tfvars + └── foo (terraform) diff --git a/cmd/infracost/testdata/generate/sibling_and_child_prefer_child/expected-tree.golden b/cmd/infracost/testdata/generate/sibling_and_child_prefer_child/expected-tree.golden new file mode 100755 index 00000000000..7e584e09799 --- /dev/null +++ b/cmd/infracost/testdata/generate/sibling_and_child_prefer_child/expected-tree.golden @@ -0,0 +1,10 @@ +└── apps (terraform) + ├── bar (terraform) + │ └── envs + │ └── prod.tfvars + ├── envs + │ ├── dev.tfvars + │ └── prod.tfvars + └── foo (terraform) + └── envs + └── dev.tfvars diff --git a/cmd/infracost/testdata/generate/sibling_and_child_prefer_sibling/expected-tree.golden b/cmd/infracost/testdata/generate/sibling_and_child_prefer_sibling/expected-tree.golden new file mode 100755 index 00000000000..21bec49e783 --- /dev/null +++ b/cmd/infracost/testdata/generate/sibling_and_child_prefer_sibling/expected-tree.golden @@ -0,0 +1,9 @@ +├── apps (terraform) +│ ├── bar (terraform) +│ ├── envs +│ │ ├── dev.tfvars +│ │ └── staging.tfvars +│ └── foo (terraform) +└── envs + ├── dev.tfvars + └── prod.tfvars diff --git a/cmd/infracost/testdata/generate/single_nested_project/expected-tree.golden b/cmd/infracost/testdata/generate/single_nested_project/expected-tree.golden new file mode 100755 index 00000000000..0796dc53d0b --- /dev/null +++ b/cmd/infracost/testdata/generate/single_nested_project/expected-tree.golden @@ -0,0 +1,2 @@ +└── app + └── foo (terraform) diff --git a/cmd/infracost/testdata/generate/single_project/expected-tree.golden b/cmd/infracost/testdata/generate/single_project/expected-tree.golden new file mode 100755 index 00000000000..4d4e920fd3a --- /dev/null +++ b/cmd/infracost/testdata/generate/single_project/expected-tree.golden @@ -0,0 +1,3 @@ +└── . (terraform) + ├── dev.tfvars + └── prod.tfvars diff --git a/cmd/infracost/testdata/generate/single_root_project/expected-tree.golden b/cmd/infracost/testdata/generate/single_root_project/expected-tree.golden new file mode 100755 index 00000000000..2f37f560b57 --- /dev/null +++ b/cmd/infracost/testdata/generate/single_root_project/expected-tree.golden @@ -0,0 +1 @@ +└── . (terraform) diff --git a/cmd/infracost/testdata/generate/terragrunt/expected-tree.golden b/cmd/infracost/testdata/generate/terragrunt/expected-tree.golden new file mode 100755 index 00000000000..e514bbe05bd --- /dev/null +++ b/cmd/infracost/testdata/generate/terragrunt/expected-tree.golden @@ -0,0 +1,5 @@ +└── apps + ├── bar (terragrunt) + ├── baz + │ └── bip (terragrunt) + └── foo (terragrunt) diff --git a/cmd/infracost/testdata/generate/terragrunt_and_terraform_mixed/expected-tree.golden b/cmd/infracost/testdata/generate/terragrunt_and_terraform_mixed/expected-tree.golden new file mode 100755 index 00000000000..f5472fbfd43 --- /dev/null +++ b/cmd/infracost/testdata/generate/terragrunt_and_terraform_mixed/expected-tree.golden @@ -0,0 +1,9 @@ +└── apps + ├── bar (terragrunt) + ├── baz + │ └── bip (terragrunt) + ├── envs + │ ├── dev.tfvars + │ └── prod.tfvars + ├── fez (terraform) + └── foo (terragrunt) diff --git a/cmd/infracost/testdata/generate/tfvar_json/expected-tree.golden b/cmd/infracost/testdata/generate/tfvar_json/expected-tree.golden new file mode 100755 index 00000000000..e789fe13d2e --- /dev/null +++ b/cmd/infracost/testdata/generate/tfvar_json/expected-tree.golden @@ -0,0 +1,7 @@ +├── terraform (terraform) +└── variables + └── env + ├── dev + │ └── tfvars.json + └── prod + └── tfvars.json diff --git a/cmd/infracost/testdata/generate/wildcard_env_names/expected-tree.golden b/cmd/infracost/testdata/generate/wildcard_env_names/expected-tree.golden new file mode 100755 index 00000000000..e08b138ffe9 --- /dev/null +++ b/cmd/infracost/testdata/generate/wildcard_env_names/expected-tree.golden @@ -0,0 +1,11 @@ +└── terraform (terraform) + └── env + ├── conf-dev-foo.tfvars + ├── conf-prod-foo.tfvars + ├── dev.tfvars + ├── ops-dev.tfvars + ├── ops-prod-bar.tfvars + ├── ops-prod-foo.tfvars + ├── other-uat.tfvars + ├── uk-uat.tfvars + └── us-uat.tfvars diff --git a/cmd/infracost/testdata/generate/with_no_directories_with_providers/expected-tree.golden b/cmd/infracost/testdata/generate/with_no_directories_with_providers/expected-tree.golden new file mode 100755 index 00000000000..7489071e90e --- /dev/null +++ b/cmd/infracost/testdata/generate/with_no_directories_with_providers/expected-tree.golden @@ -0,0 +1,4 @@ +└── modules + ├── bar (terraform) + ├── baz (terraform) + └── foo (terraform) diff --git a/examples/terraform/modules/foo/main.tf b/examples/terraform/modules/foo/main.tf new file mode 100644 index 00000000000..943a512ff64 --- /dev/null +++ b/examples/terraform/modules/foo/main.tf @@ -0,0 +1,26 @@ + +locals { + foo = [ + "a", + "b", + "c", + ] +} + +resource "aws_instance" "web_app" { + for_each = toset(local.foo) + + ami = "ami-674cbc1e" + instance_type = "m5.4xlarge" # <<<<< Try changing this to m5.8xlarge to compare the costs + + root_block_device { + volume_size = 50 + } + + ebs_block_device { + device_name = "my_data" + volume_type = "io1" # <<<<< Try changing this to gp2 to compare costs + volume_size = 1000 + iops = 800 + } +} diff --git a/internal/hcl/project_locator.go b/internal/hcl/project_locator.go index b00841dede0..ddff9b28535 100644 --- a/internal/hcl/project_locator.go +++ b/internal/hcl/project_locator.go @@ -578,6 +578,99 @@ func buildVarFileEnvNames(root *TreeNode, e *EnvFileMatcher, preferFolderNameFor }) } +// PrintTree prints a simplified tree of the detected projects and var files +// similar to a linux tree command. +func (t *TreeNode) PrintTree() string { + var sb strings.Builder + t.printTreeWithPrefix(&sb, "", true) + return sb.String() +} + +// printTreeWithPrefix is a helper function that recursively prints the tree structure +func (t *TreeNode) printTreeWithPrefix(sb *strings.Builder, prefix string, isLast bool) { + // Skip the root node as it's just a container + if t.Name != "root" { + if isLast { + sb.WriteString(prefix + "└── ") + } else { + sb.WriteString(prefix + "├── ") + } + + sb.WriteString(t.Name) + if t.RootPath != nil { + if t.RootPath.IsTerragrunt { + sb.WriteString(" (terragrunt") + } else { + sb.WriteString(" (terraform") + } + + if t.RootPath.IsParentTerragruntConfig { + sb.WriteString(", parent") + } + sb.WriteString(")") + + if len(t.RootPath.TerraformVarFiles) > 0 { + t.printVarFiles(t.RootPath.TerraformVarFiles, sb, prefix, isLast) + } + } + + if t.TerraformVarFiles != nil { + t.printVarFiles(t.TerraformVarFiles.Files, sb, prefix, isLast) + } + sb.WriteString("\n") + } + + newPrefix := prefix + if t.Name != "root" { + if isLast { + newPrefix += " " + } else { + newPrefix += "│ " + } + } + + // Sort children alphabetically + sortedChildren := make([]*TreeNode, len(t.Children)) + copy(sortedChildren, t.Children) + sort.Slice(sortedChildren, func(i, j int) bool { + return sortedChildren[i].Name < sortedChildren[j].Name + }) + + for i, child := range sortedChildren { + child.printTreeWithPrefix(sb, newPrefix, i == len(sortedChildren)-1) + } +} + +func (t *TreeNode) printVarFiles(files []RootPathVarFile, sb *strings.Builder, prefix string, isLast bool) { + sb.WriteString("\n") + sortedFiles := make([]RootPathVarFile, len(files)) + copy(sortedFiles, files) + + sort.Slice(sortedFiles, func(i, j int) bool { + return sortedFiles[i].Name < sortedFiles[j].Name + }) + + for i, f := range sortedFiles { + lastVarFile := i == len(sortedFiles)-1 + + varFilePrefix := prefix + "│ " + if isLast { + varFilePrefix = prefix + " " + } + + suffix := "\n" + + if lastVarFile { + suffix = "" + sb.WriteString(varFilePrefix + "└── ") + } else { + sb.WriteString(varFilePrefix + "├── ") + } + + sb.WriteString(f.Name + suffix) + } +} + // AddPath adds a path to the tree, this will create any missing nodes in the tree. func (t *TreeNode) AddPath(path RootPath) { dir, _ := filepath.Rel(path.StartingPath, path.DetectedPath) @@ -1194,7 +1287,7 @@ func (r *RootPath) AddVarFiles(v *VarFiles) { // FindRootModules returns a list of all directories that contain a full // Terraform project under the given fullPath. This list excludes any Terraform // modules that have been found (if they have been called by a Module source). -func (p *ProjectLocator) FindRootModules(startingPath string) []RootPath { +func (p *ProjectLocator) FindRootModules(startingPath string) ([]RootPath, string) { p.basePath, _ = filepath.Abs(startingPath) p.modules = make(map[string]struct{}) p.projectDuplicates = make(map[string]bool) @@ -1224,7 +1317,7 @@ func (p *ProjectLocator) FindRootModules(startingPath string) []RootPath { IsTerragrunt: p.wdContainsTerragrunt, TerraformVarFiles: p.discoveredVarFiles[startingPath], }, - } + }, "" } p.findTerragruntDirs(startingPath) @@ -1279,6 +1372,8 @@ func (p *ProjectLocator) FindRootModules(startingPath string) []RootPath { } node := CreateTreeNode(startingPath, projects, p.discoveredVarFiles, p.envMatcher, p.preferFolderNameForEnv) + tree := node.PrintTree() + node.AssociateChildVarFiles() node.AssociateSiblingVarFiles() node.AssociateParentVarFiles() @@ -1301,7 +1396,7 @@ func (p *ProjectLocator) FindRootModules(startingPath string) []RootPath { return paths[i].DetectedPath < paths[j].DetectedPath }) - return paths + return paths, tree } // excludeEnvFromPaths filters car files from the paths based on the path overrides. diff --git a/internal/providers/detect.go b/internal/providers/detect.go index 84b30d7209f..d33c2736925 100644 --- a/internal/providers/detect.go +++ b/internal/providers/detect.go @@ -20,6 +20,7 @@ import ( type DetectionOutput struct { Providers []schema.Provider RootModules int + Tree string } // Detect returns a list of providers for the given path. Multiple returned @@ -88,9 +89,11 @@ func Detect(ctx *config.RunContext, project *config.Project, includePastResource } pl := hcl.NewProjectLocator(logging.Logger, locatorConfig) - rootPaths := pl.FindRootModules(project.Path) + rootPaths, tree := pl.FindRootModules(project.Path) if len(rootPaths) == 0 { - return &DetectionOutput{}, fmt.Errorf("could not detect path type for '%s'", project.Path) + return &DetectionOutput{ + Tree: tree, + }, fmt.Errorf("could not detect path type for '%s'", project.Path) } var autoProviders []schema.Provider @@ -110,7 +113,7 @@ func Detect(ctx *config.RunContext, project *config.Project, includePastResource } } - return &DetectionOutput{Providers: autoProviders, RootModules: len(rootPaths)}, nil + return &DetectionOutput{Providers: autoProviders, RootModules: len(rootPaths), Tree: tree}, nil } // configFileRootToProvider returns a provider for the given root path which is diff --git a/internal/providers/terraform/hcl_provider_test.go b/internal/providers/terraform/hcl_provider_test.go index 40360d10310..3a78146bc14 100644 --- a/internal/providers/terraform/hcl_provider_test.go +++ b/internal/providers/terraform/hcl_provider_test.go @@ -207,7 +207,7 @@ func TestHCLProvider_LoadPlanJSON(t *testing.T) { startingPath = "." } - mods := pl.FindRootModules(startingPath) + mods, _ := pl.FindRootModules(startingPath) options := []hcl.Option{hcl.OptionWithBlockBuilder( hcl.BlockBuilder{ MockFunc: func(a *hcl.Attribute) cty.Value { From daea6b24aad987a7114791c180c1d027a7c76ba4 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Mon, 30 Dec 2024 10:44:11 +0000 Subject: [PATCH 75/81] chore: update to go 1.23 --- Dockerfile | 2 +- Dockerfile.ci | 2 +- go.mod | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 753c4552e36..f14c88fe53c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.22 as builder +FROM golang:1.23 as builder ARG ARCH=linux ARG DEFAULT_TERRAFORM_VERSION=0.15.5 diff --git a/Dockerfile.ci b/Dockerfile.ci index a6b446cb94a..5799d573289 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM golang:1.22 as builder +FROM golang:1.23 as builder ARG ARCH=linux64 diff --git a/go.mod b/go.mod index ac9d2f9b810..8b41d6bafd5 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/infracost/infracost -go 1.22 +go 1.23 require ( github.com/Masterminds/goutils v1.1.1 // indirect From af1add832ce1ba16998fa0a7d056ec9b760b31a8 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Mon, 30 Dec 2024 10:49:30 +0000 Subject: [PATCH 76/81] chore: dockerfile linting --- Dockerfile | 8 ++++---- Dockerfile.ci | 8 ++++---- Dockerfile.wolfi | 8 ++++---- Dockerfile.wolfi-ci | 8 ++++---- Dockerfile.wolfi-latest | 6 +++--- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index f14c88fe53c..f58d28a225f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.23 as builder +FROM golang:1.23 AS builder ARG ARCH=linux ARG DEFAULT_TERRAFORM_VERSION=0.15.5 @@ -6,8 +6,8 @@ ARG TERRAGRUNT_VERSION=0.31.8 # Set Environment Variables SHELL ["/bin/bash", "-c"] -ENV HOME /app -ENV CGO_ENABLED 0 +ENV HOME=/app +ENV CGO_ENABLED=0 # Install Packages RUN apt-get update -q && apt-get -y install unzip @@ -44,7 +44,7 @@ RUN NO_DIRTY=true make build RUN chmod +x /app/build/infracost # Application -FROM alpine:3.16 as app +FROM alpine:3.16 AS app # Tools needed for running diffs in CI integrations RUN apk --no-cache add ca-certificates openssl openssh-client curl git bash diff --git a/Dockerfile.ci b/Dockerfile.ci index 5799d573289..e0f8be8c893 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,11 +1,11 @@ -FROM golang:1.23 as builder +FROM golang:1.23 AS builder ARG ARCH=linux64 # Set Environment Variables SHELL ["/bin/bash", "-c"] -ENV HOME /app -ENV CGO_ENABLED 0 +ENV HOME=/app +ENV CGO_ENABLED=0 WORKDIR /app @@ -15,7 +15,7 @@ RUN NO_DIRTY=true make build RUN chmod +x /app/build/infracost # Application -FROM alpine:3.16 as app +FROM alpine:3.16 AS app # Tools needed for running diffs in CI integrations RUN apk --no-cache add bash curl git nodejs npm openssh-client diff --git a/Dockerfile.wolfi b/Dockerfile.wolfi index 0a830dfe112..d3839f68be7 100644 --- a/Dockerfile.wolfi +++ b/Dockerfile.wolfi @@ -1,4 +1,4 @@ -FROM cgr.dev/chainguard/go:latest-dev as builder +FROM cgr.dev/chainguard/go:latest-dev AS builder ARG ARCH=linux ARG DEFAULT_TERRAFORM_VERSION=0.15.5 @@ -6,8 +6,8 @@ ARG TERRAGRUNT_VERSION=0.31.8 # Set Environment Variables SHELL ["/bin/bash", "-c"] -ENV HOME /app -ENV CGO_ENABLED 0 +ENV HOME=/app +ENV CGO_ENABLED=0 RUN apk add bash wget # Install latest of each Terraform version after 0.12 as we don't support versions before that @@ -42,7 +42,7 @@ RUN NO_DIRTY=true make build RUN chmod +x /app/build/infracost # Application -FROM cgr.dev/chainguard/wolfi-base as app +FROM cgr.dev/chainguard/wolfi-base AS app # Tools needed for running diffs in CI integrations RUN apk --no-cache add ca-certificates openssl openssh-client curl git bash jq diff --git a/Dockerfile.wolfi-ci b/Dockerfile.wolfi-ci index 2dcab95cc93..a50f60ab92b 100644 --- a/Dockerfile.wolfi-ci +++ b/Dockerfile.wolfi-ci @@ -1,11 +1,11 @@ -FROM cgr.dev/chainguard/go as builder +FROM cgr.dev/chainguard/go AS builder ARG ARCH=linux64 # Set Environment Variables SHELL ["/bin/bash", "-c"] -ENV HOME /app -ENV CGO_ENABLED 0 +ENV HOME=/app +ENV CGO_ENABLED=0 WORKDIR /app @@ -15,7 +15,7 @@ RUN NO_DIRTY=true make build RUN chmod +x /app/build/infracost # Application -FROM cgr.dev/chainguard/wolfi-base as app +FROM cgr.dev/chainguard/wolfi-base AS app # Tools needed for running diffs in CI integrations RUN apk --no-cache add bash curl git nodejs openssh-client jq diff --git a/Dockerfile.wolfi-latest b/Dockerfile.wolfi-latest index f33c53bbd9f..ba8740b13cf 100644 --- a/Dockerfile.wolfi-latest +++ b/Dockerfile.wolfi-latest @@ -1,8 +1,8 @@ -FROM cgr.dev/chainguard/go:latest-dev as builder +FROM cgr.dev/chainguard/go:latest-dev AS builder -ENV HOME /app +ENV HOME=/app WORKDIR /app -ENV CGO_ENABLED 0 +ENV CGO_ENABLED=0 # Build Application COPY . . From 7eb769d9e096e278fdc9351b27bb87a166fa4b95 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Mon, 30 Dec 2024 15:20:24 +0000 Subject: [PATCH 77/81] fix: pass currency in for Pricing API requests PReviously we were setting this on the Pricing API Client but that is re-used, so this was not being set properly between tests --- internal/apiclient/pricing.go | 21 ++- internal/apiclient/pricing_test.go | 5 +- internal/prices/prices.go | 7 +- .../dynamodb_table_china_test.golden | 26 +-- .../lambda_function_china_test.golden | 14 +- .../rds_cluster_china_test.golden | 4 +- .../s3_bucket_china_test.golden | 148 +++++++++--------- 7 files changed, 112 insertions(+), 113 deletions(-) diff --git a/internal/apiclient/pricing.go b/internal/apiclient/pricing.go index 626ea19234c..6dd201be505 100644 --- a/internal/apiclient/pricing.go +++ b/internal/apiclient/pricing.go @@ -33,7 +33,6 @@ var ( type PricingAPIClient struct { APIClient - Currency string EventsDisabled bool cacheFile string @@ -96,11 +95,6 @@ func NewPricingAPIClient(ctx *config.RunContext) *PricingAPIClient { return nil } - currency := ctx.Config.Currency - if currency == "" { - currency = "USD" - } - tlsConfig := tls.Config{} // nolint: gosec if ctx.Config.TLSCACertFile != "" { @@ -141,7 +135,6 @@ func NewPricingAPIClient(ctx *config.RunContext) *PricingAPIClient { apiKey: ctx.Config.APIKey, uuid: ctx.UUID(), }, - Currency: currency, EventsDisabled: ctx.Config.EventsDisabled, } @@ -254,7 +247,11 @@ func (c *PricingAPIClient) AddEvent(name string, env map[string]interface{}) err return err } -func (c *PricingAPIClient) buildQuery(product *schema.ProductFilter, price *schema.PriceFilter) GraphQLQuery { +func (c *PricingAPIClient) buildQuery(product *schema.ProductFilter, price *schema.PriceFilter, currency string) GraphQLQuery { + if currency == "" { + currency = "USD" + } + v := map[string]interface{}{} v["productFilter"] = product v["priceFilter"] = price @@ -268,14 +265,14 @@ func (c *PricingAPIClient) buildQuery(product *schema.ProductFilter, price *sche } } } - `, c.Currency) + `, currency) return GraphQLQuery{query, v} } // BatchRequests batches all the queries for these resources so we can use less GraphQL requests // Use PriceQueryKeys to keep track of which query maps to which sub-resource and price component. -func (c *PricingAPIClient) BatchRequests(resources []*schema.Resource, batchSize int) []BatchRequest { +func (c *PricingAPIClient) BatchRequests(resources []*schema.Resource, batchSize int, currency string) []BatchRequest { reqs := make([]BatchRequest, 0) keys := make([]PriceQueryKey, 0) @@ -284,13 +281,13 @@ func (c *PricingAPIClient) BatchRequests(resources []*schema.Resource, batchSize for _, r := range resources { for _, component := range r.CostComponents { keys = append(keys, PriceQueryKey{r, component}) - queries = append(queries, c.buildQuery(component.ProductFilter, component.PriceFilter)) + queries = append(queries, c.buildQuery(component.ProductFilter, component.PriceFilter, currency)) } for _, subresource := range r.FlattenedSubResources() { for _, component := range subresource.CostComponents { keys = append(keys, PriceQueryKey{subresource, component}) - queries = append(queries, c.buildQuery(component.ProductFilter, component.PriceFilter)) + queries = append(queries, c.buildQuery(component.ProductFilter, component.PriceFilter, currency)) } } } diff --git a/internal/apiclient/pricing_test.go b/internal/apiclient/pricing_test.go index 63273df5325..ce5b1fc6ab2 100644 --- a/internal/apiclient/pricing_test.go +++ b/internal/apiclient/pricing_test.go @@ -71,7 +71,6 @@ func TestPricingAPIClient_PerformRequest(t *testing.T) { apiKey: ctx.Config.APIKey, uuid: ctx.UUID(), }, - Currency: "USD", } initCache(ctx, c) @@ -113,12 +112,12 @@ func TestPricingAPIClient_PerformRequest(t *testing.T) { }, }, } - q := c.buildQuery(cachedProduct, nil) + q := c.buildQuery(cachedProduct, nil, "USD") k, err := hashstructure.Hash(q, hashstructure.FormatV2, nil) assert.NoError(t, err) c.cache.Add(k, cacheValue{Result: gjson.Parse(`{"data":{"products":[{"prices":[{"priceHash":"cached-ee3dd7e4624338037ca6fea0933a662f","USD":"0.1250000000"}]}]}`), ExpiresAt: time.Now().Add(time.Hour)}) - batches := c.BatchRequests(resources, 100) + batches := c.BatchRequests(resources, 100, "USD") result, err := c.PerformRequest(batches[0]) assert.Len(t, requestMap, 1, "invalid number of requests made to pricing API") diff --git a/internal/prices/prices.go b/internal/prices/prices.go index b82f5e960ba..ad9db3c37c7 100644 --- a/internal/prices/prices.go +++ b/internal/prices/prices.go @@ -213,7 +213,7 @@ func (p *PriceFetcher) getPricesConcurrent(resources []*schema.Resource) error { numWorkers = 16 } - reqs := p.client.BatchRequests(resources, batchSize) + reqs := p.client.BatchRequests(resources, batchSize, p.runCtx.Config.Currency) numJobs := len(reqs) jobs := make(chan apiclient.BatchRequest, numJobs) @@ -279,7 +279,10 @@ type productPrice struct { } func (p *PriceFetcher) setCostComponentPrice(result apiclient.PriceQueryResult) { - currency := p.client.Currency + currency := p.runCtx.Config.Currency + if currency == "" { + currency = "USD" + } if result.CostComponent.CustomPrice() != nil { logging.Logger.Debug().Msgf("Using user-defined custom price %v for %s %s.", *result.CostComponent.CustomPrice(), result.Resource.Name, result.CostComponent.Name) diff --git a/internal/providers/terraform/aws/testdata/dynamodb_table_china_test/dynamodb_table_china_test.golden b/internal/providers/terraform/aws/testdata/dynamodb_table_china_test/dynamodb_table_china_test.golden index ff5a3f39e42..c75fe9ac45a 100644 --- a/internal/providers/terraform/aws/testdata/dynamodb_table_china_test/dynamodb_table_china_test.golden +++ b/internal/providers/terraform/aws/testdata/dynamodb_table_china_test/dynamodb_table_china_test.golden @@ -1,16 +1,16 @@ - Name Monthly Qty Unit Monthly Cost (CNY) - - aws_dynamodb_table.my_dynamodb_table_china - ├─ Write capacity unit (WCU) 20 WCU 11.63 元 - ├─ Read capacity unit (RCU) 30 RCU 3.49 元 - ├─ Data storage Monthly cost depends on usage: 0.30 元 per GB - ├─ Point-In-Time Recovery (PITR) backup storage Monthly cost depends on usage: 0.24 元 per GB - ├─ On-demand backup storage Monthly cost depends on usage: 0.12 元 per GB - ├─ Table data restored Monthly cost depends on usage: 0.18 元 per GB - └─ Streams read request unit (sRRU) Monthly cost depends on usage: 0.0000002444 元 per sRRUs - - OVERALL TOTAL (CNY) 15.11 元 + Name Monthly Qty Unit Monthly Cost (CNY) + + aws_dynamodb_table.my_dynamodb_table_china + ├─ Write capacity unit (WCU) 20 WCU 84.68 元 + ├─ Read capacity unit (RCU) 30 RCU 25.40 元 + ├─ Data storage Monthly cost depends on usage: 2.20 元 per GB + ├─ Point-In-Time Recovery (PITR) backup storage Monthly cost depends on usage: 1.76 元 per GB + ├─ On-demand backup storage Monthly cost depends on usage: 0.88 元 per GB + ├─ Table data restored Monthly cost depends on usage: 1.32 元 per GB + └─ Streams read request unit (sRRU) Monthly cost depends on usage: 0.00000178 元 per sRRUs + + OVERALL TOTAL (CNY) 110.08 元 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -21,5 +21,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ 15 元 ┃ - ┃ 15 元 ┃ +┃ main ┃ 110 元 ┃ - ┃ 110 元 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/aws/testdata/lambda_function_china_test/lambda_function_china_test.golden b/internal/providers/terraform/aws/testdata/lambda_function_china_test/lambda_function_china_test.golden index d9c3c81731c..7d56a4b2226 100644 --- a/internal/providers/terraform/aws/testdata/lambda_function_china_test/lambda_function_china_test.golden +++ b/internal/providers/terraform/aws/testdata/lambda_function_china_test/lambda_function_china_test.golden @@ -2,17 +2,17 @@ Name Monthly Qty Unit Monthly Cost (CNY) aws_lambda_function.lambda_china_with_usage - └─ Requests 0.1 1M requests 0.02 元 * + └─ Requests 0.1 1M requests 0.14 元 * aws_lambda_function.lambda_china_with_usage_arm - └─ Requests 0.1 1M requests 0.02 元 * + └─ Requests 0.1 1M requests 0.14 元 * aws_lambda_function.lambda_china - ├─ Requests Monthly cost depends on usage: 0.19 元 per 1M requests - ├─ Ephemeral storage Monthly cost depends on usage: 0.0000000316 元 per GB-seconds - └─ Duration Monthly cost depends on usage: 0.0000155804 元 per GB-seconds + ├─ Requests Monthly cost depends on usage: 1.36 元 per 1M requests + ├─ Ephemeral storage Monthly cost depends on usage: 0.0000002303 元 per GB-seconds + └─ Duration Monthly cost depends on usage: 0.000113477 元 per GB-seconds - OVERALL TOTAL (CNY) 0.04 元 + OVERALL TOTAL (CNY) 0.27 元 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -23,5 +23,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ 0.00 元 ┃ 0.04 元 ┃ 0.04 元 ┃ +┃ main ┃ 0.00 元 ┃ 0.27 元 ┃ 0.27 元 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file diff --git a/internal/providers/terraform/aws/testdata/rds_cluster_china_test/rds_cluster_china_test.golden b/internal/providers/terraform/aws/testdata/rds_cluster_china_test/rds_cluster_china_test.golden index 2455793ae5a..7288856ef37 100644 --- a/internal/providers/terraform/aws/testdata/rds_cluster_china_test/rds_cluster_china_test.golden +++ b/internal/providers/terraform/aws/testdata/rds_cluster_china_test/rds_cluster_china_test.golden @@ -2,8 +2,8 @@ Name Monthly Qty Unit Monthly Cost (CNY) aws_rds_cluster.mysql_china - ├─ Storage Monthly cost depends on usage: 0.091991 元 per GB - ├─ I/O requests Monthly cost depends on usage: 0.18 元 per 1M requests + ├─ Storage Monthly cost depends on usage: 0.67 元 per GB + ├─ I/O requests Monthly cost depends on usage: 1.33 元 per 1M requests ├─ Backup storage not found └─ Snapshot export not found diff --git a/internal/providers/terraform/aws/testdata/s3_bucket_china_test/s3_bucket_china_test.golden b/internal/providers/terraform/aws/testdata/s3_bucket_china_test/s3_bucket_china_test.golden index f842cc129d9..81a58302236 100644 --- a/internal/providers/terraform/aws/testdata/s3_bucket_china_test/s3_bucket_china_test.golden +++ b/internal/providers/terraform/aws/testdata/s3_bucket_china_test/s3_bucket_china_test.golden @@ -1,77 +1,77 @@ - Name Monthly Qty Unit Monthly Cost (CNY) - - aws_s3_bucket.bucket_china_with_usage - ├─ Standard - │ ├─ Storage 10,000 GB 267.74 元 * - │ ├─ PUT, COPY, POST, LIST requests 10 1k requests 0.01 元 * - │ ├─ GET, SELECT, and all other requests 10 1k requests 0.00 元 * - │ ├─ Select data scanned 10,000 GB 21.87 元 * - │ └─ Select data returned 10,000 GB 7.83 元 * - ├─ Intelligent tiering - │ ├─ Storage (frequent access) 20,000 GB 535.47 元 * - │ ├─ Storage (infrequent access) 20,000 GB 367.33 元 * - │ ├─ Storage (archive access) 20,000 GB 82.54 元 * - │ ├─ Storage (deep archive access) 20,000 GB 36.69 元 * - │ ├─ Monitoring and automation 20 1k objects 0.05 元 * - │ ├─ PUT, COPY, POST, LIST requests 20 1k requests 0.01 元 * - │ ├─ GET, SELECT, and all other requests 20 1k requests 0.00 元 * - │ ├─ Lifecycle transition 20 1k requests 0.17 元 * - │ ├─ Select data scanned 20,000 GB 43.74 元 * - │ ├─ Select data returned 20,000 GB 15.65 元 * - │ └─ Early delete (within 30 days) 20,000 GB 535.47 元 * - ├─ Standard - infrequent access - │ ├─ Storage 30,000 GB 551.00 元 * - │ ├─ PUT, COPY, POST, LIST requests 30 1k requests 0.26 元 * - │ ├─ GET, SELECT, and all other requests 30 1k requests 0.03 元 * - │ ├─ Lifecycle transition 30 1k requests 0.26 元 * - │ ├─ Retrievals 30,000 GB 262.38 元 * - │ ├─ Select data scanned 30,000 GB 65.62 元 * - │ └─ Select data returned 30,000 GB 262.38 元 * - ├─ One zone - infrequent access - │ ├─ Storage 40,000 GB 587.73 元 * - │ ├─ PUT, COPY, POST, LIST requests 40 1k requests 0.35 元 * - │ ├─ GET, SELECT, and all other requests 40 1k requests 0.03 元 * - │ ├─ Lifecycle transition 40 1k requests 0.35 元 * - │ ├─ Retrievals 40,000 GB 349.84 元 * - │ ├─ Select data scanned 40,000 GB 87.49 元 * - │ └─ Select data returned 40,000 GB 349.84 元 * - ├─ Glacier flexible retrieval - │ ├─ Storage 50,000 GB 206.36 元 * - │ ├─ PUT, COPY, POST, LIST requests 50 1k requests 1.54 元 * - │ ├─ GET, SELECT, and all other requests 50 1k requests 0.01 元 * - │ ├─ Lifecycle transition 50 1k requests 1.54 元 * - │ ├─ Retrieval requests (standard) 50 1k requests 1.54 元 * - │ ├─ Retrievals (standard) 50,000 GB 457.90 元 * - │ ├─ Select data scanned (standard) 50,000 GB 384.44 元 * - │ ├─ Select data returned (standard) 50,000 GB 457.90 元 * - │ ├─ Retrieval requests (expedited) 50 1k requests 457.90 元 * - │ ├─ Retrievals (expedited) 50,000 GB 1,373.69 元 * - │ ├─ Select data scanned (expedited) 50,000 GB 961.79 元 * - │ ├─ Select data returned (expedited) 50,000 GB 1,373.69 元 * - │ ├─ Select data scanned (bulk) 50,000 GB 46.00 元 * - │ ├─ Select data returned (bulk) 50,000 GB 114.65 元 * - │ └─ Early delete (within 90 days) 50,000 GB 206.36 元 * - └─ Glacier deep archive - ├─ Storage 60,000 GB 110.06 元 * - ├─ PUT, COPY, POST, LIST requests 60 1k requests 3.69 元 * - ├─ GET, SELECT, and all other requests 60 1k requests 0.01 元 * - ├─ Lifecycle transition 60 1k requests 3.69 元 * - ├─ Retrieval requests (standard) 60 1k requests 6.15 元 * - ├─ Retrievals (standard) 60,000 GB 1,098.95 元 * - ├─ Retrieval requests (bulk) 60 1k requests 1.37 元 * - ├─ Retrievals (bulk) 60,000 GB 275.15 元 * - └─ Early delete (within 180 days) 60,000 GB 110.06 元 * - - aws_s3_bucket.bucket_china - └─ Standard - ├─ Storage Monthly cost depends on usage: 0.0267735 元 per GB - ├─ PUT, COPY, POST, LIST requests Monthly cost depends on usage: 0.0006179 元 per 1k requests - ├─ GET, SELECT, and all other requests Monthly cost depends on usage: 0.000206 元 per 1k requests - ├─ Select data scanned Monthly cost depends on usage: 0.002187189 元 per GB - └─ Select data returned Monthly cost depends on usage: 0.00078261 元 per GB - - OVERALL TOTAL (CNY) 12,086.54 元 + Name Monthly Qty Unit Monthly Cost (CNY) + + aws_s3_bucket.bucket_china_with_usage + ├─ Standard + │ ├─ Storage 10,000 GB 1,950.00 元 * + │ ├─ PUT, COPY, POST, LIST requests 10 1k requests 0.05 元 * + │ ├─ GET, SELECT, and all other requests 10 1k requests 0.02 元 * + │ ├─ Select data scanned 10,000 GB 159.30 元 * + │ └─ Select data returned 10,000 GB 57.00 元 * + ├─ Intelligent tiering + │ ├─ Storage (frequent access) 20,000 GB 3,900.00 元 * + │ ├─ Storage (infrequent access) 20,000 GB 2,675.40 元 * + │ ├─ Storage (archive access) 20,000 GB 601.20 元 * + │ ├─ Storage (deep archive access) 20,000 GB 267.20 元 * + │ ├─ Monitoring and automation 20 1k objects 0.33 元 * + │ ├─ PUT, COPY, POST, LIST requests 20 1k requests 0.09 元 * + │ ├─ GET, SELECT, and all other requests 20 1k requests 0.03 元 * + │ ├─ Lifecycle transition 20 1k requests 1.27 元 * + │ ├─ Select data scanned 20,000 GB 318.60 元 * + │ ├─ Select data returned 20,000 GB 114.00 元 * + │ └─ Early delete (within 30 days) 20,000 GB 3,900.00 元 * + ├─ Standard - infrequent access + │ ├─ Storage 30,000 GB 4,013.10 元 * + │ ├─ PUT, COPY, POST, LIST requests 30 1k requests 1.91 元 * + │ ├─ GET, SELECT, and all other requests 30 1k requests 0.19 元 * + │ ├─ Lifecycle transition 30 1k requests 1.91 元 * + │ ├─ Retrievals 30,000 GB 1,911.00 元 * + │ ├─ Select data scanned 30,000 GB 477.90 元 * + │ └─ Select data returned 30,000 GB 1,911.00 元 * + ├─ One zone - infrequent access + │ ├─ Storage 40,000 GB 4,280.64 元 * + │ ├─ PUT, COPY, POST, LIST requests 40 1k requests 2.55 元 * + │ ├─ GET, SELECT, and all other requests 40 1k requests 0.25 元 * + │ ├─ Lifecycle transition 40 1k requests 2.55 元 * + │ ├─ Retrievals 40,000 GB 2,548.00 元 * + │ ├─ Select data scanned 40,000 GB 637.20 元 * + │ └─ Select data returned 40,000 GB 2,548.00 元 * + ├─ Glacier flexible retrieval + │ ├─ Storage 50,000 GB 1,503.00 元 * + │ ├─ PUT, COPY, POST, LIST requests 50 1k requests 11.20 元 * + │ ├─ GET, SELECT, and all other requests 50 1k requests 0.08 元 * + │ ├─ Lifecycle transition 50 1k requests 11.20 元 * + │ ├─ Retrieval requests (standard) 50 1k requests 11.20 元 * + │ ├─ Retrievals (standard) 50,000 GB 3,335.00 元 * + │ ├─ Select data scanned (standard) 50,000 GB 2,800.00 元 * + │ ├─ Select data returned (standard) 50,000 GB 3,335.00 元 * + │ ├─ Retrieval requests (expedited) 50 1k requests 3,335.00 元 * + │ ├─ Retrievals (expedited) 50,000 GB 10,005.00 元 * + │ ├─ Select data scanned (expedited) 50,000 GB 7,005.00 元 * + │ ├─ Select data returned (expedited) 50,000 GB 10,005.00 元 * + │ ├─ Select data scanned (bulk) 50,000 GB 335.00 元 * + │ ├─ Select data returned (bulk) 50,000 GB 835.00 元 * + │ └─ Early delete (within 90 days) 50,000 GB 1,503.00 元 * + └─ Glacier deep archive + ├─ Storage 60,000 GB 801.60 元 * + ├─ PUT, COPY, POST, LIST requests 60 1k requests 26.87 元 * + ├─ GET, SELECT, and all other requests 60 1k requests 0.09 元 * + ├─ Lifecycle transition 60 1k requests 26.87 元 * + ├─ Retrieval requests (standard) 60 1k requests 44.78 元 * + ├─ Retrievals (standard) 60,000 GB 8,004.00 元 * + ├─ Retrieval requests (bulk) 60 1k requests 10.01 元 * + ├─ Retrievals (bulk) 60,000 GB 2,004.00 元 * + └─ Early delete (within 180 days) 60,000 GB 801.60 元 * + + aws_s3_bucket.bucket_china + └─ Standard + ├─ Storage Monthly cost depends on usage: 0.20 元 per GB + ├─ PUT, COPY, POST, LIST requests Monthly cost depends on usage: 0.0045 元 per 1k requests + ├─ GET, SELECT, and all other requests Monthly cost depends on usage: 0.0015 元 per 1k requests + ├─ Select data scanned Monthly cost depends on usage: 0.01593 元 per GB + └─ Select data returned Monthly cost depends on usage: 0.0057 元 per GB + + OVERALL TOTAL (CNY) 88,030.17 元 *Usage costs can be estimated by updating Infracost Cloud settings, see docs for other options. @@ -82,5 +82,5 @@ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ ┃ Project ┃ Baseline cost ┃ Usage cost* ┃ Total cost ┃ ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━╋━━━━━━━━━━━━┫ -┃ main ┃ 0.00 元 ┃ 12,087 元 ┃ 12,087 元 ┃ +┃ main ┃ 0.00 元 ┃ 88,030 元 ┃ 88,030 元 ┃ ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━┻━━━━━━━━━━━━┛ \ No newline at end of file From c2c0718b6f3b22926ef582abf6fd43c095546ddb Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Mon, 30 Dec 2024 16:13:13 +0000 Subject: [PATCH 78/81] chore: use debug for non-important errors --- internal/hcl/modules/fetch.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/hcl/modules/fetch.go b/internal/hcl/modules/fetch.go index a99fd473ce9..9f94e8affb8 100644 --- a/internal/hcl/modules/fetch.go +++ b/internal/hcl/modules/fetch.go @@ -95,7 +95,8 @@ func (p *PackageFetcher) Fetch(moduleAddr string, dest string) error { } if err != nil { - p.logger.Warn().Msgf("error fetching module %s from local cache: %s", util.RedactUrl(moduleAddr), err) + // Log the error at debug level because the local cache might be invalid and in that case we just refetch it so it's not a big deal + p.logger.Debug().Msgf("error fetching module %s from local cache: %s", util.RedactUrl(moduleAddr), err) } fetched, err = p.fetchFromRemoteCache(moduleAddr, dest) @@ -106,7 +107,8 @@ func (p *PackageFetcher) Fetch(moduleAddr string, dest string) error { } if err != nil { - p.logger.Warn().Msgf("error fetching module %s from remote cache: %s", util.RedactUrl(moduleAddr), err) + // Log the error at debug level because the remote cache might be invalid and in that case we just refetch it so it's not a big deal + p.logger.Debug().Msgf("error fetching module %s from remote cache: %s", util.RedactUrl(moduleAddr), err) } p.logger.Trace().Msgf("cache miss: %s", moduleAddr) From 0b2c3e3c3efe9547b97f6f76c80317e64f471518 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Mon, 30 Dec 2024 17:39:49 +0000 Subject: [PATCH 79/81] fix(aws): nil ptr errors in `aws_lb` --- internal/resources/aws/lb.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/internal/resources/aws/lb.go b/internal/resources/aws/lb.go index 64549f2b68a..61c382ff0f9 100644 --- a/internal/resources/aws/lb.go +++ b/internal/resources/aws/lb.go @@ -52,14 +52,24 @@ func (r *LB) BuildResource() *schema.Resource { if r.ActiveConnections != nil { activeConnections := decimal.NewFromInt(*r.ActiveConnections) activeConnectionsLCU = decimalPtr(activeConnections.Div(decimal.NewFromInt(3000))) - maxLCU = decimalPtr(decimal.Max(*maxLCU, *activeConnectionsLCU)) + + if maxLCU == nil { + maxLCU = activeConnectionsLCU + } else { + maxLCU = decimalPtr(decimal.Max(*maxLCU, *activeConnectionsLCU)) + } } var processedBytesLCU *decimal.Decimal if r.ProcessedBytesGB != nil { processedBytes := decimal.NewFromFloat(*r.ProcessedBytesGB) processedBytesLCU = decimalPtr(processedBytes.Div(decimal.NewFromInt(1))) - maxLCU = decimalPtr(decimal.Max(*maxLCU, *processedBytesLCU)) + + if maxLCU == nil { + maxLCU = processedBytesLCU + } else { + maxLCU = decimalPtr(decimal.Max(*maxLCU, *processedBytesLCU)) + } } var costComponents []*schema.CostComponent @@ -69,7 +79,12 @@ func (r *LB) BuildResource() *schema.Resource { if r.RuleEvaluations != nil && maxLCU != nil { ruleEvaluations := decimal.NewFromInt(*r.RuleEvaluations) ruleEvaluationsLCU = ruleEvaluations.Div(decimal.NewFromInt(1000)) - maxLCU = decimalPtr(decimal.Max(*maxLCU, ruleEvaluationsLCU)) + + if maxLCU == nil { + maxLCU = &ruleEvaluationsLCU + } else { + maxLCU = decimalPtr(decimal.Max(*maxLCU, ruleEvaluationsLCU)) + } } costComponents = r.applicationLBCostComponents(maxLCU) From 61fb061aae2ec1a2a45bdb4eb426a914d6d8775d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 31 Dec 2024 10:08:28 +0000 Subject: [PATCH 80/81] chore(deps): bump filippo.io/age from 1.0.0 to 1.2.1 Bumps [filippo.io/age](https://github.com/FiloSottile/age) from 1.0.0 to 1.2.1. - [Release notes](https://github.com/FiloSottile/age/releases) - [Commits](https://github.com/FiloSottile/age/compare/v1.0.0...v1.2.1) --- updated-dependencies: - dependency-name: filippo.io/age dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 8 ++++---- go.sum | 22 ++++++++++++---------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 8b41d6bafd5..90290048d15 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/tidwall/gjson v1.17.0 github.com/zclconf/go-cty v1.14.1 golang.org/x/crypto v0.31.0 // indirect - golang.org/x/mod v0.17.0 + golang.org/x/mod v0.18.0 gopkg.in/go-playground/assert.v1 v1.2.1 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 @@ -80,7 +80,7 @@ require ( github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect golang.org/x/text v0.21.0 - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + golang.org/x/tools v0.22.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect ) @@ -129,7 +129,7 @@ require ( cloud.google.com/go/compute/metadata v0.3.0 // indirect cloud.google.com/go/iam v0.13.0 // indirect dario.cat/mergo v1.0.0 // indirect - filippo.io/age v1.0.0 // indirect + filippo.io/age v1.2.1 // indirect github.com/Azure/azure-sdk-for-go v63.3.0+incompatible // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.26 // indirect @@ -262,7 +262,7 @@ require ( github.com/yashtewari/glob-intersection v0.1.0 // indirect github.com/zclconf/go-cty-yaml v1.0.3 go.opencensus.io v0.24.0 // indirect - golang.org/x/net v0.25.0 // indirect + golang.org/x/net v0.26.0 // indirect google.golang.org/api v0.114.0 google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect diff --git a/go.sum b/go.sum index 522f18a3a04..62d102a166c 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +c2sp.org/CCTV/age v0.0.0-20240306222714-3ec4d716e805 h1:u2qwJeEvnypw+OCPUHmoZE3IqwfuN5kgDfo5MLzpNM0= +c2sp.org/CCTV/age v0.0.0-20240306222714-3ec4d716e805/go.mod h1:FomMrUJ2Lxt5jCLmZkG3FHa72zUprnhd3v/Z18Snm4w= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -190,8 +192,8 @@ cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoIS dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -filippo.io/age v1.0.0 h1:V6q14n0mqYU3qKFkZ6oOaF9oXneOviS3ubXsSVBRSzc= -filippo.io/age v1.0.0/go.mod h1:PaX+Si/Sd5G8LgfCwldsSba3H1DDQZhIhFGkhbHaBq8= +filippo.io/age v1.2.1 h1:X0TZjehAZylOIj4DubWYU1vWQxv9bJpo+Uu2/LGhi1o= +filippo.io/age v1.2.1/go.mod h1:JL9ew2lTN+Pyft4RiNGguFfOpewKwSHm5ayKD/A4004= github.com/Azure/azure-sdk-for-go v45.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v47.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v51.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= @@ -1054,8 +1056,8 @@ github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= @@ -1288,8 +1290,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1353,8 +1355,8 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1584,8 +1586,8 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From d0cf963fbdc7bb9d4cccac5655ed2f2fe77834f6 Mon Sep 17 00:00:00 2001 From: Alistair Scott Date: Tue, 31 Dec 2024 10:22:44 +0000 Subject: [PATCH 81/81] chore: update golang.org/x/net --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 90290048d15..5c113c682dc 100644 --- a/go.mod +++ b/go.mod @@ -262,7 +262,7 @@ require ( github.com/yashtewari/glob-intersection v0.1.0 // indirect github.com/zclconf/go-cty-yaml v1.0.3 go.opencensus.io v0.24.0 // indirect - golang.org/x/net v0.26.0 // indirect + golang.org/x/net v0.33.0 // indirect google.golang.org/api v0.114.0 google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect diff --git a/go.sum b/go.sum index 62d102a166c..fbf8a799399 100644 --- a/go.sum +++ b/go.sum @@ -1355,8 +1355,8 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=