From 9c6cc6d321d87cb4c0dfa8010617fef36526c118 Mon Sep 17 00:00:00 2001 From: Jari Voutilainen Date: Wed, 18 Jun 2025 14:13:17 +0300 Subject: [PATCH 1/6] AV-2443: Update cloudstorage and show validation errors in the UI --- ckan/ckanext/ckanext-cloudstorage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/ckanext/ckanext-cloudstorage b/ckan/ckanext/ckanext-cloudstorage index d34f9da0c..51f7c3afb 160000 --- a/ckan/ckanext/ckanext-cloudstorage +++ b/ckan/ckanext/ckanext-cloudstorage @@ -1 +1 @@ -Subproject commit d34f9da0c9dd4f2e09c1bdf8d15770cbaf5b334b +Subproject commit 51f7c3afb487c39439ac3b2fdcc5f4f5c9f393dc From 47fa4ee196a8a034010ded71c3d847762f7c7db1 Mon Sep 17 00:00:00 2001 From: Jari Voutilainen Date: Wed, 18 Jun 2025 14:38:43 +0300 Subject: [PATCH 2/6] Update cloudstorage --- ckan/ckanext/ckanext-cloudstorage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/ckanext/ckanext-cloudstorage b/ckan/ckanext/ckanext-cloudstorage index 51f7c3afb..bee260927 160000 --- a/ckan/ckanext/ckanext-cloudstorage +++ b/ckan/ckanext/ckanext-cloudstorage @@ -1 +1 @@ -Subproject commit 51f7c3afb487c39439ac3b2fdcc5f4f5c9f393dc +Subproject commit bee2609279c684e9f7ef7d18281162c51f31bee2 From a41565f2b95319eec38a4b0ebd0f875b25e8f334 Mon Sep 17 00:00:00 2001 From: Jari Voutilainen Date: Wed, 18 Jun 2025 14:47:27 +0300 Subject: [PATCH 3/6] Update cloudstorage variables in ckan config --- cdk/lib/ckan-stack.ts | 6 ++---- ckan/templates/production.ini.j2 | 1 - docker/.env.ckan.local | 3 +-- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/cdk/lib/ckan-stack.ts b/cdk/lib/ckan-stack.ts index 5de99267f..e780c3692 100644 --- a/cdk/lib/ckan-stack.ts +++ b/cdk/lib/ckan-stack.ts @@ -375,8 +375,7 @@ export class CkanStack extends Stack { ckanContainerEnv['CKAN_CLOUDSTORAGE_ENABLED'] = 'true'; ckanContainerEnv['CKAN_CLOUDSTORAGE_DRIVER'] = pCkanCloudstorageDriver.stringValue; ckanContainerEnv['CKAN_CLOUDSTORAGE_CONTAINER_NAME'] = pCkanCloudstorageContainerName.stringValue; - ckanContainerEnv['CKAN_CLOUDSTORAGE_USE_SECURE_URLS'] = pCkanCloudstorageUseSecureUrls.stringValue; - ckanContainerEnv['CKAN_CLOUDSTORAGE_AWS_USE_BOTO3_SESSIONS'] = '1'; + ckanContainerEnv['CKAN_CLOUDSTORAGE_USE_SECURE_URLS'] = true ckanContainerEnv['CKAN_CLOUDSTORAGE_DRIVER_OPTIONS'] = ''; ckanTaskPolicyAllowCloudstorage = new iam.PolicyStatement({ @@ -393,8 +392,7 @@ export class CkanStack extends Stack { ckanContainerEnv['CKAN_CLOUDSTORAGE_ENABLED'] = 'false'; ckanContainerEnv['CKAN_CLOUDSTORAGE_DRIVER'] = ''; ckanContainerEnv['CKAN_CLOUDSTORAGE_CONTAINER_NAME'] = ''; - ckanContainerEnv['CKAN_CLOUDSTORAGE_USE_SECURE_URLS'] = ''; - ckanContainerEnv['CKAN_CLOUDSTORAGE_AWS_USE_BOTO3_SESSIONS'] = '0'; + ckanContainerEnv['CKAN_CLOUDSTORAGE_USE_SECURE_URLS'] = false; } const ckanLogGroup = new logs.LogGroup(this, 'ckanLogGroup', { diff --git a/ckan/templates/production.ini.j2 b/ckan/templates/production.ini.j2 index be5427dfc..e359113e4 100644 --- a/ckan/templates/production.ini.j2 +++ b/ckan/templates/production.ini.j2 @@ -204,7 +204,6 @@ ckanext.reminder.email_field = maintainer_email ckanext.cloudstorage.driver = {{ environ('CKAN_CLOUDSTORAGE_DRIVER') }} ckanext.cloudstorage.container_name = {{ environ('CKAN_CLOUDSTORAGE_CONTAINER_NAME') }} ckanext.cloudstorage.use_secure_urls = {{ environ('CKAN_CLOUDSTORAGE_USE_SECURE_URLS') }} -ckanext.cloudstorage.aws_use_boto3_sessions = {{ environ('CKAN_CLOUDSTORAGE_AWS_USE_BOTO3_SESSIONS') }} ckanext.cloudstorage.driver_options = {{ environ('CKAN_CLOUDSTORAGE_DRIVER_OPTIONS') }} ckanext.cloudstorage.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet ods application/vnd.oasis.opendocument.spreadsheet ckan.datapusher.formats = nonexistingformat diff --git a/docker/.env.ckan.local b/docker/.env.ckan.local index acffd777b..f08f8798e 100644 --- a/docker/.env.ckan.local +++ b/docker/.env.ckan.local @@ -28,8 +28,7 @@ CKAN_CLOUDSTORAGE_ENABLED=false CKAN_CLOUDSTORAGE_DRIVER="S3_EU_WEST" CKAN_CLOUDSTORAGE_DRIVER_OPTIONS="" CKAN_CLOUDSTORAGE_CONTAINER_NAME="" -CKAN_CLOUDSTORAGE_USE_SECURE_URLS=1 -CKAN_CLOUDSTORAGE_AWS_USE_BOTO3_SESSIONS=1 +CKAN_CLOUDSTORAGE_USE_SECURE_URLS=true AWS_ACCESS_KEY_ID="" AWS_SECRET_ACCESS_KEY="" ZULIP_API_URL= From b4b9faf104840bee8e9eeb8714ad461f92bc3485 Mon Sep 17 00:00:00 2001 From: Jari Voutilainen Date: Wed, 18 Jun 2025 15:14:11 +0300 Subject: [PATCH 4/6] Dont use secure urls locally --- docker/.env.ckan.local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/.env.ckan.local b/docker/.env.ckan.local index f08f8798e..ab2daf381 100644 --- a/docker/.env.ckan.local +++ b/docker/.env.ckan.local @@ -28,7 +28,7 @@ CKAN_CLOUDSTORAGE_ENABLED=false CKAN_CLOUDSTORAGE_DRIVER="S3_EU_WEST" CKAN_CLOUDSTORAGE_DRIVER_OPTIONS="" CKAN_CLOUDSTORAGE_CONTAINER_NAME="" -CKAN_CLOUDSTORAGE_USE_SECURE_URLS=true +CKAN_CLOUDSTORAGE_USE_SECURE_URLS=false AWS_ACCESS_KEY_ID="" AWS_SECRET_ACCESS_KEY="" ZULIP_API_URL= From 87b9da409adb3ed2c7602082581320ec834756a8 Mon Sep 17 00:00:00 2001 From: Jari Voutilainen Date: Wed, 18 Jun 2025 15:14:26 +0300 Subject: [PATCH 5/6] parameters need to be strings --- cdk/lib/ckan-stack.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdk/lib/ckan-stack.ts b/cdk/lib/ckan-stack.ts index e780c3692..6b324483b 100644 --- a/cdk/lib/ckan-stack.ts +++ b/cdk/lib/ckan-stack.ts @@ -375,7 +375,7 @@ export class CkanStack extends Stack { ckanContainerEnv['CKAN_CLOUDSTORAGE_ENABLED'] = 'true'; ckanContainerEnv['CKAN_CLOUDSTORAGE_DRIVER'] = pCkanCloudstorageDriver.stringValue; ckanContainerEnv['CKAN_CLOUDSTORAGE_CONTAINER_NAME'] = pCkanCloudstorageContainerName.stringValue; - ckanContainerEnv['CKAN_CLOUDSTORAGE_USE_SECURE_URLS'] = true + ckanContainerEnv['CKAN_CLOUDSTORAGE_USE_SECURE_URLS'] = 'true' ckanContainerEnv['CKAN_CLOUDSTORAGE_DRIVER_OPTIONS'] = ''; ckanTaskPolicyAllowCloudstorage = new iam.PolicyStatement({ @@ -392,7 +392,7 @@ export class CkanStack extends Stack { ckanContainerEnv['CKAN_CLOUDSTORAGE_ENABLED'] = 'false'; ckanContainerEnv['CKAN_CLOUDSTORAGE_DRIVER'] = ''; ckanContainerEnv['CKAN_CLOUDSTORAGE_CONTAINER_NAME'] = ''; - ckanContainerEnv['CKAN_CLOUDSTORAGE_USE_SECURE_URLS'] = false; + ckanContainerEnv['CKAN_CLOUDSTORAGE_USE_SECURE_URLS'] = 'false'; } const ckanLogGroup = new logs.LogGroup(this, 'ckanLogGroup', { From 3fae9a1dc6025b09494665b2a5b3f485541ade76 Mon Sep 17 00:00:00 2001 From: Jari Voutilainen Date: Wed, 18 Jun 2025 15:44:21 +0300 Subject: [PATCH 6/6] provide credentials for cloudstorage and use secure urls --- ckan/ckanext/ckanext-cloudstorage | 2 +- docker/.env.ckan.local | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/ckanext/ckanext-cloudstorage b/ckan/ckanext/ckanext-cloudstorage index bee260927..5f53f07bc 160000 --- a/ckan/ckanext/ckanext-cloudstorage +++ b/ckan/ckanext/ckanext-cloudstorage @@ -1 +1 @@ -Subproject commit bee2609279c684e9f7ef7d18281162c51f31bee2 +Subproject commit 5f53f07bc28556f7140b5eaf2e75d2e2178ffe8e diff --git a/docker/.env.ckan.local b/docker/.env.ckan.local index ab2daf381..f08f8798e 100644 --- a/docker/.env.ckan.local +++ b/docker/.env.ckan.local @@ -28,7 +28,7 @@ CKAN_CLOUDSTORAGE_ENABLED=false CKAN_CLOUDSTORAGE_DRIVER="S3_EU_WEST" CKAN_CLOUDSTORAGE_DRIVER_OPTIONS="" CKAN_CLOUDSTORAGE_CONTAINER_NAME="" -CKAN_CLOUDSTORAGE_USE_SECURE_URLS=false +CKAN_CLOUDSTORAGE_USE_SECURE_URLS=true AWS_ACCESS_KEY_ID="" AWS_SECRET_ACCESS_KEY="" ZULIP_API_URL=