From 353438dbd300641e125cd52a300c9a261a03a69f Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Thu, 4 Feb 2021 01:34:12 -0500 Subject: [PATCH 1/7] feat: allow builders to be detected based on repository contents Rather than hardcode two builders, allow builders to specify a `builder-detect` trigger. This trigger can be used to specify if the builder should or should not be used for an application. Each builder takes stdin and can decide if it wants to emit it or emit it's own image source type. If the final value is empty, then Dokku will default to herokuish (and cnb once that is stable). In addition, a future change may allow users to manually specify a builder in the case they wish to override the choice selected by Dokku. This change enables users to build custom builder plugins and have those plugins used for building an image asset. By way of example, an enterprising user could create a `builder-lambda` based on lambci, and then pair this with a scheduler plugin that updates a lambda function on AWS. Alternatively, a user might decide they wish to place their Dockerfile in a specific directory for their applications - such as an `_infrastructure` directory - and create a plugin to override how that is detected within Dokku. --- docs/development/plugin-triggers.md | 18 ++++ plugins/20_events/builder-detect | 1 + plugins/builder-cnb/builder-detect | 22 ++++ plugins/builder-dockerfile/builder-detect | 30 ++++++ plugins/builder-herokuish/builder-detect | 27 +++++ plugins/builder-herokuish/functions | 38 +++++++ plugins/git/functions | 17 ++- plugins/tar/functions | 17 ++- tests/unit/builder.bats | 121 ++++++++++++++++++++++ 9 files changed, 271 insertions(+), 20 deletions(-) create mode 120000 plugins/20_events/builder-detect create mode 100755 plugins/builder-cnb/builder-detect create mode 100755 plugins/builder-dockerfile/builder-detect create mode 100755 plugins/builder-herokuish/builder-detect create mode 100755 plugins/builder-herokuish/functions create mode 100644 tests/unit/builder.bats diff --git a/docs/development/plugin-triggers.md b/docs/development/plugin-triggers.md index 61d05c60300..fa8505b9bab 100644 --- a/docs/development/plugin-triggers.md +++ b/docs/development/plugin-triggers.md @@ -143,6 +143,24 @@ set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x # TODO ``` +### `builder-detect` + +- Description: Allows overriding the auto-detected `herokuish` builder in favor of a custom one. Dockerfile gets lowest builder precedence. +- Invoked by: `dokku deploy` +- Arguments: `$APP` `$SOURCECODE_WORK_DIR` +- Example: + +```shell +#!/usr/bin/env bash + +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x +APP="$1"; SOURCECODE_WORK_DIR="$2" + +if [[ -f "$SOURCECODE_WORK_DIR/project.toml" ]]; then + echo -n "cnb" +fi +``` + ### `builder-create-dokku-image` - Description: Allows modification of the configured dokku-image diff --git a/plugins/20_events/builder-detect b/plugins/20_events/builder-detect new file mode 120000 index 00000000000..5178a749ff6 --- /dev/null +++ b/plugins/20_events/builder-detect @@ -0,0 +1 @@ +hook \ No newline at end of file diff --git a/plugins/builder-cnb/builder-detect b/plugins/builder-cnb/builder-detect new file mode 100755 index 00000000000..65678e55d57 --- /dev/null +++ b/plugins/builder-cnb/builder-detect @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +source "$PLUGIN_AVAILABLE_PATH/config/functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +trigger-builder-cnb-builder-detect() { + declare desc="builder-cnb builder-detect plugin trigger" + declare trigger="builder-detect" + declare APP="$1" SOURCECODE_WORK_DIR="$2" + + if [[ -f "$SOURCECODE_WORK_DIR/project.toml" ]]; then + echo "cnb" + return + fi + + if [[ "$(config_get "$APP" DOKKU_CNB_EXPERIMENTAL || true)" == "1" ]]; then + echo "cnb" + return + fi +} + +trigger-builder-cnb-builder-detect "$@" diff --git a/plugins/builder-dockerfile/builder-detect b/plugins/builder-dockerfile/builder-detect new file mode 100755 index 00000000000..1def88e9fa4 --- /dev/null +++ b/plugins/builder-dockerfile/builder-detect @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +source "$PLUGIN_AVAILABLE_PATH/config/functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +trigger-builder-dockerfile-builder-detect() { + declare desc="builder-dockerfile builder-detect plugin trigger" + declare trigger="builder-detect" + declare APP="$1" SOURCECODE_WORK_DIR="$2" + + # buildpacks always win against dockerfile detection + # that includes cnb + if [[ -f "$SOURCECODE_WORK_DIR/project.toml" ]]; then + return + fi + + # buildpacks always win against dockerfile detection + # that includes cnb + if [[ "$(config_get "$APP" DOKKU_CNB_EXPERIMENTAL || true)" == "1" ]]; then + echo "cnb" + return + fi + + if [[ -f "$SOURCECODE_WORK_DIR/Dockerfile" ]]; then + echo "dockerfile" + return + fi +} + +trigger-builder-dockerfile-builder-detect "$@" diff --git a/plugins/builder-herokuish/builder-detect b/plugins/builder-herokuish/builder-detect new file mode 100755 index 00000000000..7ee618691b6 --- /dev/null +++ b/plugins/builder-herokuish/builder-detect @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +source "$PLUGIN_AVAILABLE_PATH/builder-herokuish/functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +trigger-builder-herokuish-builder-detect() { + declare desc="builder-herokuish builder-detect plugin trigger" + declare trigger="builder-detect" + declare APP="$1" SOURCECODE_WORK_DIR="$2" + + if fn-has-buildpacks-file "$SOURCECODE_WORK_DIR"; then + echo "herokuish" + return + fi + + if fn-has-buildpack-dotenv "$SOURCECODE_WORK_DIR"; then + echo "herokuish" + return + fi + + if fn-has-buildpack-env "$APP"; then + echo "herokuish" + return + fi +} + +trigger-builder-herokuish-builder-detect "$@" diff --git a/plugins/builder-herokuish/functions b/plugins/builder-herokuish/functions new file mode 100755 index 00000000000..8452214a131 --- /dev/null +++ b/plugins/builder-herokuish/functions @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +source "$PLUGIN_AVAILABLE_PATH/config/functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +fn-has-buildpacks-file() { + declare SOURCECODE_WORK_DIR="$1" + + if [[ -f "$SOURCECODE_WORK_DIR/.buildpacks" ]]; then + return 0 + fi + + return 1 +} + +fn-has-buildpack-dotenv() { + declare SOURCECODE_WORK_DIR="$1" + + if [[ ! -f "$SOURCECODE_WORK_DIR/.env" ]]; then + return 1 + fi + + if grep -q BUILDPACK_URL "$SOURCECODE_WORK_DIR/.env"; then + return 0 + fi + + return 1 +} + +fn-has-buildpack-env() { + declare APP="$1" + + if [[ -n $(config_get "$APP" BUILDPACK_URL || true) ]]; then + return 0 + fi + + return 1 +} diff --git a/plugins/git/functions b/plugins/git/functions index 114c2897122..a67b795e1b8 100755 --- a/plugins/git/functions +++ b/plugins/git/functions @@ -49,18 +49,15 @@ git_build_app_repo() { git_trigger_build() { declare desc="triggers the actual build process for a given app within a directory at a particular revision" declare APP="$1" TMP_WORK_DIR="$2" REV="$3" + local BUILDER plugn trigger post-extract "$APP" "$TMP_WORK_DIR" "$REV" - if [[ -f Dockerfile ]] && [[ "$( - [[ -f .env ]] && grep -q BUILDPACK_URL .env - echo $? - )" != "0" ]] && [[ ! -f ".buildpacks" ]] && [[ -z $(config_get "$APP" BUILDPACK_URL || true) ]]; then - plugn trigger pre-receive-app "$APP" "dockerfile" "$TMP_WORK_DIR" "$REV" - dokku_receive "$APP" "dockerfile" "$TMP_WORK_DIR" - else - plugn trigger pre-receive-app "$APP" "herokuish" "$TMP_WORK_DIR" "$REV" - dokku_receive "$APP" "herokuish" "$TMP_WORK_DIR" - fi + + BUILDER="$(plugn trigger builder-detect "$APP" "$TMP_WORK_DIR" | tail -n1 || true)" + [[ -z "$BUILDER" ]] && BUILDER="herokuish" + + plugn trigger pre-receive-app "$APP" "$BUILDER" "$TMP_WORK_DIR" "$REV" + dokku_receive "$APP" "$BUILDER" "$TMP_WORK_DIR" } git_deploy_branch() { diff --git a/plugins/tar/functions b/plugins/tar/functions index 9a8302c9936..c95d2930336 100755 --- a/plugins/tar/functions +++ b/plugins/tar/functions @@ -41,18 +41,15 @@ tar_build() { tar_trigger_build() { declare desc="triggers the actual build process for a given app within a directory at a particular revision" declare APP="$1" TMP_WORK_DIR="$2" REV="$3" + local BUILDER plugn trigger post-extract "$APP" "$TMP_WORK_DIR" "$REV" - if [[ -f Dockerfile ]] && [[ "$( - [[ -f .env ]] && grep -q BUILDPACK_URL .env - echo $? - )" != "0" ]] && [[ ! -f ".buildpacks" ]] && [[ -z $(config_get "$APP" BUILDPACK_URL || true) ]]; then - plugn trigger pre-receive-app "$APP" "dockerfile" "$TMP_WORK_DIR" "$REV" - dokku_receive "$APP" "dockerfile" "$TMP_WORK_DIR" - else - plugn trigger pre-receive-app "$APP" "herokuish" "$TMP_WORK_DIR" "$REV" - dokku_receive "$APP" "herokuish" "$TMP_WORK_DIR" - fi + + BUILDER="$(plugn trigger builder-detect "$APP" "$TMP_WORK_DIR" | tail -n1 || true)" + [[ -z "$BUILDER" ]] && BUILDER="herokuish" + + plugn trigger pre-receive-app "$APP" "$BUILDER" "$TMP_WORK_DIR" "$REV" + dokku_receive "$APP" "$BUILDER" "$TMP_WORK_DIR" } cmd-tar-in() { diff --git a/tests/unit/builder.bats b/tests/unit/builder.bats new file mode 100644 index 00000000000..d6f620c4ef1 --- /dev/null +++ b/tests/unit/builder.bats @@ -0,0 +1,121 @@ +#!/usr/bin/env bats + +load test_helper + +setup() { + create_app +} + +teardown() { + destroy_app +} + +@test "(builder) builder-detect [cnb]" { + local TMP=$(mktemp -d "/tmp/dokku.me.XXXXX") + trap 'popd &>/dev/null || true; rm -rf "$TMP"' INT TERM + + # test project.toml + run touch "$TMP/project.toml" + echo "output: $output" + echo "status: $status" + assert_success + + chown -R dokku:dokku "$TMP" + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | tail -n1" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "cnb" + + sudo rm -rf $TMP/* + echo "output: $output" + echo "status: $status" + assert_success + + # test DOKKU_CNB_EXPERIMENTAL env var + run /bin/bash -c "dokku config:set $TEST_APP DOKKU_CNB_EXPERIMENTAL=1" + echo "output: $output" + echo "status: $status" + assert_success + + chown -R dokku:dokku "$TMP" + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | tail -n1" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "cnb" +} + +@test "(builder) builder-detect [dockerfile]" { + local TMP=$(mktemp -d "/tmp/dokku.me.XXXXX") + trap 'popd &>/dev/null || true; rm -rf "$TMP"' INT TERM + + run touch "$TMP/Dockerfile" + echo "output: $output" + echo "status: $status" + assert_success + + chown -R dokku:dokku "$TMP" + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | tail -n1" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "dockerfile" +} + +@test "(builder) builder-detect [herokuish]" { + local TMP=$(mktemp -d "/tmp/dokku.me.XXXXX") + trap 'popd &>/dev/null || true; rm -rf "$TMP"' INT TERM + + touch "$TMP/Dockerfile" + + # test buildpacks + chown -R dokku:dokku "$TMP" + run /bin/bash -c "touch $TMP/.buildpacks" + echo "output: $output" + echo "status: $status" + assert_success + + chown -R dokku:dokku "$TMP" + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | tail -n1" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "herokuish" + + sudo rm -rf $TMP/* + echo "output: $output" + echo "status: $status" + assert_success + + # test .env + run /bin/bash -c "echo BUILDPACK_URL=null > $TMP/.env" + echo "output: $output" + echo "status: $status" + assert_success + + chown -R dokku:dokku "$TMP" + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | tail -n1" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "herokuish" + + sudo rm -rf $TMP/* + echo "output: $output" + echo "status: $status" + assert_success + + # test BUILDPACK_URL env var + run /bin/bash -c "dokku config:set $TEST_APP BUILDPACK_URL=null" + echo "output: $output" + echo "status: $status" + assert_success + + chown -R dokku:dokku "$TMP" + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | tail -n1" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "herokuish" +} From 6462dd891c7743aa3258662e0bc6846b198bbd5d Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 28 Feb 2021 15:29:20 -0500 Subject: [PATCH 2/7] chore: rename builder-cnb to builder-pack --- docs/appendices/0.24.0-migration-guide.md | 1 + plugins/builder-cnb/plugin.toml | 4 ---- plugins/{builder-cnb => builder-pack}/builder-build | 6 +++--- plugins/{builder-cnb => builder-pack}/builder-detect | 6 +++--- plugins/{builder-cnb => builder-pack}/builder-release | 6 +++--- plugins/builder-pack/plugin.toml | 4 ++++ 6 files changed, 14 insertions(+), 13 deletions(-) delete mode 100644 plugins/builder-cnb/plugin.toml rename plugins/{builder-cnb => builder-pack}/builder-build (90%) rename plugins/{builder-cnb => builder-pack}/builder-detect (73%) rename plugins/{builder-cnb => builder-pack}/builder-release (81%) create mode 100644 plugins/builder-pack/plugin.toml diff --git a/docs/appendices/0.24.0-migration-guide.md b/docs/appendices/0.24.0-migration-guide.md index d4fa0a7ea19..f55d81c2fd0 100644 --- a/docs/appendices/0.24.0-migration-guide.md +++ b/docs/appendices/0.24.0-migration-guide.md @@ -3,3 +3,4 @@ ## Changes - The commands `proxy:enable`, `proxy:disable` and `proxy:build-config` now support the `--all` flag in addition to general parallelism. +- The `builder-cnb` plugin has been renamed `builder-pack` diff --git a/plugins/builder-cnb/plugin.toml b/plugins/builder-cnb/plugin.toml deleted file mode 100644 index 65e913fe15e..00000000000 --- a/plugins/builder-cnb/plugin.toml +++ /dev/null @@ -1,4 +0,0 @@ -[plugin] -description = "dokku core builder-cnb plugin" -version = "0.23.9" -[plugin.config] diff --git a/plugins/builder-cnb/builder-build b/plugins/builder-pack/builder-build similarity index 90% rename from plugins/builder-cnb/builder-build rename to plugins/builder-pack/builder-build index 22f98900f9f..10854f8682c 100755 --- a/plugins/builder-cnb/builder-build +++ b/plugins/builder-pack/builder-build @@ -4,8 +4,8 @@ set -eo pipefail source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" source "$PLUGIN_AVAILABLE_PATH/config/functions" -trigger-builder-cnb-builder-build() { - declare desc="builder-cnb builder-build plugin trigger" +trigger-builder-pack-builder-build() { + declare desc="builder-pack builder-build plugin trigger" declare trigger="builder-build" declare BUILDER_TYPE="$1" APP="$2" SOURCECODE_WORK_DIR="$3" @@ -42,4 +42,4 @@ trigger-builder-cnb-builder-build() { plugn trigger post-build-cnb "$APP" "$SOURCECODE_WORK_DIR" } -trigger-builder-cnb-builder-build "$@" +trigger-builder-pack-builder-build "$@" diff --git a/plugins/builder-cnb/builder-detect b/plugins/builder-pack/builder-detect similarity index 73% rename from plugins/builder-cnb/builder-detect rename to plugins/builder-pack/builder-detect index 65678e55d57..b40d1d52ffc 100755 --- a/plugins/builder-cnb/builder-detect +++ b/plugins/builder-pack/builder-detect @@ -3,8 +3,8 @@ source "$PLUGIN_AVAILABLE_PATH/config/functions" set -eo pipefail [[ $DOKKU_TRACE ]] && set -x -trigger-builder-cnb-builder-detect() { - declare desc="builder-cnb builder-detect plugin trigger" +trigger-builder-pack-builder-detect() { + declare desc="builder-pack builder-detect plugin trigger" declare trigger="builder-detect" declare APP="$1" SOURCECODE_WORK_DIR="$2" @@ -19,4 +19,4 @@ trigger-builder-cnb-builder-detect() { fi } -trigger-builder-cnb-builder-detect "$@" +trigger-builder-pack-builder-detect "$@" diff --git a/plugins/builder-cnb/builder-release b/plugins/builder-pack/builder-release similarity index 81% rename from plugins/builder-cnb/builder-release rename to plugins/builder-pack/builder-release index e3bb97d6b5c..95e02836951 100755 --- a/plugins/builder-cnb/builder-release +++ b/plugins/builder-pack/builder-release @@ -3,8 +3,8 @@ set -eo pipefail [[ $DOKKU_TRACE ]] && set -x source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions" -trigger-builder-cnb-builder-release() { - declare desc="builder-cnb builder-release plugin trigger" +trigger-builder-pack-builder-release() { + declare desc="builder-pack builder-release plugin trigger" declare trigger="builder-release" declare BUILDER_TYPE="$1" APP="$2" IMAGE_TAG="$3" @@ -19,4 +19,4 @@ trigger-builder-cnb-builder-release() { plugn trigger post-release-cnb "$APP" "$IMAGE_TAG" } -trigger-builder-cnb-builder-release "$@" +trigger-builder-pack-builder-release "$@" diff --git a/plugins/builder-pack/plugin.toml b/plugins/builder-pack/plugin.toml new file mode 100644 index 00000000000..3ff193f2a17 --- /dev/null +++ b/plugins/builder-pack/plugin.toml @@ -0,0 +1,4 @@ +[plugin] +description = "dokku core builder-pack plugin" +version = "0.23.9" +[plugin.config] From 43b9d9d4b6fbd3bc36c30da9b0643f491e6d1038 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 28 Feb 2021 15:31:56 -0500 Subject: [PATCH 3/7] chore: drop unused code The builder-pack rename makes this unnecessary. --- plugins/builder-dockerfile/builder-detect | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/plugins/builder-dockerfile/builder-detect b/plugins/builder-dockerfile/builder-detect index 1def88e9fa4..d14de16aa22 100755 --- a/plugins/builder-dockerfile/builder-detect +++ b/plugins/builder-dockerfile/builder-detect @@ -8,19 +8,6 @@ trigger-builder-dockerfile-builder-detect() { declare trigger="builder-detect" declare APP="$1" SOURCECODE_WORK_DIR="$2" - # buildpacks always win against dockerfile detection - # that includes cnb - if [[ -f "$SOURCECODE_WORK_DIR/project.toml" ]]; then - return - fi - - # buildpacks always win against dockerfile detection - # that includes cnb - if [[ "$(config_get "$APP" DOKKU_CNB_EXPERIMENTAL || true)" == "1" ]]; then - echo "cnb" - return - fi - if [[ -f "$SOURCECODE_WORK_DIR/Dockerfile" ]]; then echo "dockerfile" return From 1ec71cd509b5f59532ab0fc945d5210387311d99 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 28 Feb 2021 16:02:22 -0500 Subject: [PATCH 4/7] refactor: switch detected builder so first one wins Also rename internal cnb references to pack (where possible). --- docs/appendices/0.24.0-migration-guide.md | 6 ++++- docs/development/plugin-triggers.md | 26 +++++++++---------- plugins/app-json/functions.go | 2 +- plugins/builder-dockerfile/builder-detect | 18 +++++++++++++ plugins/builder-pack/builder-build | 6 ++--- plugins/builder-pack/builder-detect | 4 +-- plugins/builder-pack/builder-release | 6 ++--- plugins/common/functions | 4 +-- plugins/git/functions | 2 +- .../scheduler-docker-local/scheduler-deploy | 2 +- plugins/tar/functions | 2 +- tests/unit/builder.bats | 18 ++++++------- 12 files changed, 59 insertions(+), 37 deletions(-) diff --git a/docs/appendices/0.24.0-migration-guide.md b/docs/appendices/0.24.0-migration-guide.md index f55d81c2fd0..fb104995d91 100644 --- a/docs/appendices/0.24.0-migration-guide.md +++ b/docs/appendices/0.24.0-migration-guide.md @@ -3,4 +3,8 @@ ## Changes - The commands `proxy:enable`, `proxy:disable` and `proxy:build-config` now support the `--all` flag in addition to general parallelism. -- The `builder-cnb` plugin has been renamed `builder-pack` +- The `builder-cnb` plugin has been renamed `builder-pack`, and all related plugin triggers have had the suffix `-cnb` changed to `-pack`. + +## Deprecations + +- The 1.0.0 release of Dokku will no longer select buildpack builders over dockerfile builders if both builders match. Instead, Dokku will choose the first builder that responds to the `builder-detect` trigger. diff --git a/docs/development/plugin-triggers.md b/docs/development/plugin-triggers.md index fa8505b9bab..da4ac32ef9b 100644 --- a/docs/development/plugin-triggers.md +++ b/docs/development/plugin-triggers.md @@ -157,7 +157,7 @@ set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x APP="$1"; SOURCECODE_WORK_DIR="$2" if [[ -f "$SOURCECODE_WORK_DIR/project.toml" ]]; then - echo -n "cnb" + echo -n "pack" fi ``` @@ -1093,12 +1093,12 @@ set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x # TODO ``` -### `post-build-cnb` +### `post-build-pack` -> Warning: The cnb plugin trigger apis are under development and may change +> Warning: The pack plugin trigger apis are under development and may change > between minor releases until the 1.0 release. -- Description: Allows you to run commands after the build image is create for a given app. Only applies to apps using cnb. +- Description: Allows you to run commands after the build image is create for a given app. Only applies to apps using pack. - Invoked by: `internal function dokku_build() (build phase)` - Arguments: `$APP` `$SOURCECODE_WORK_DIR` - Example: @@ -1326,14 +1326,14 @@ APP="$1"; IMAGE_TAG="$2"; IMAGE=$(get_app_image_name $APP $IMAGE_TAG) # TODO ``` -### `post-release-cnb` +### `post-release-pack` -> Warning: The cnb plugin trigger apis are under development and may change +> Warning: The pack plugin trigger apis are under development and may change > between minor releases until the 1.0 release. > Warning: Image mutation in this trigger may result in an invalid run state, and is heavily discouraged. -- Description: Allows you to run commands after environment variables are set for the release step of the deploy. Only applies to apps using cnb. +- Description: Allows you to run commands after environment variables are set for the release step of the deploy. Only applies to apps using pack. - Invoked by: `internal function dokku_release() (release phase)` - Arguments: `$APP $IMAGE_TAG` - Example: @@ -1400,12 +1400,12 @@ set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x # TODO ``` -### `pre-build-cnb` +### `pre-build-pack` -> Warning: The cnb plugin trigger apis are under development and may change +> Warning: The pack plugin trigger apis are under development and may change > between minor releases until the 1.0 release. -- Description: Allows you to run commands before the build image is created for a given app. For instance, this can be useful to add env vars to your container. Only applies to apps using cnb. +- Description: Allows you to run commands before the build image is created for a given app. For instance, this can be useful to add env vars to your container. Only applies to apps using pack. - Invoked by: `internal function dokku_build() (build phase)` - Arguments: `$APP` `$SOURCECODE_WORK_DIR` - Example: @@ -1557,12 +1557,12 @@ docker commit "${DOCKER_COMMIT_LABEL_ARGS[@]}" $CID $IMAGE >/dev/null ``` -### `pre-release-cnb` +### `pre-release-pack` -> Warning: The cnb plugin trigger apis are under development and may change +> Warning: The pack plugin trigger apis are under development and may change > between minor releases until the 1.0 release. -- Description: Allows you to run commands before environment variables are set for the release step of the deploy. Only applies to apps using cnb. +- Description: Allows you to run commands before environment variables are set for the release step of the deploy. Only applies to apps using pack. - Invoked by: `internal function dokku_release() (release phase)` - Arguments: `$APP $IMAGE_TAG` - Example: diff --git a/plugins/app-json/functions.go b/plugins/app-json/functions.go index e70c5f28160..65bc8bd3a08 100644 --- a/plugins/app-json/functions.go +++ b/plugins/app-json/functions.go @@ -188,7 +188,7 @@ func executeScript(appName string, image string, imageTag string, phase string) if isHerokuishImage { imageSourceType = "herokuish" } else if isCnbImage { - imageSourceType = "cnb" + imageSourceType = "pack" } cacheDir := fmt.Sprintf("%s/cache", common.AppRoot(appName)) diff --git a/plugins/builder-dockerfile/builder-detect b/plugins/builder-dockerfile/builder-detect index d14de16aa22..43cd9fcdbed 100755 --- a/plugins/builder-dockerfile/builder-detect +++ b/plugins/builder-dockerfile/builder-detect @@ -8,6 +8,24 @@ trigger-builder-dockerfile-builder-detect() { declare trigger="builder-detect" declare APP="$1" SOURCECODE_WORK_DIR="$2" + # hack: unfortunately our legacy code requires that buildpacks + # are detected before dockerfile support is detected + # as such, we need to force-check the herokuish and pack + # builders before allowing the dockerfile check to succeed + # in a future release, we may drop this hack, but for now, + # such is life + if [[ -f "$PLUGIN_ENABLED_PATH/builder-herokuish/builder-detect" ]]; then + if [[ -n "$($PLUGIN_ENABLED_PATH/builder-herokuish/builder-detect "$APP" "$SOURCECODE_WORK_DIR")" ]]; then + return + fi + fi + + if [[ -f "$PLUGIN_ENABLED_PATH/builder-pack/builder-detect" ]]; then + if [[ -n "$($PLUGIN_ENABLED_PATH/builder-pack/builder-detect "$APP" "$SOURCECODE_WORK_DIR")" ]]; then + return + fi + fi + if [[ -f "$SOURCECODE_WORK_DIR/Dockerfile" ]]; then echo "dockerfile" return diff --git a/plugins/builder-pack/builder-build b/plugins/builder-pack/builder-build index 10854f8682c..d1dcd96f8cf 100755 --- a/plugins/builder-pack/builder-build +++ b/plugins/builder-pack/builder-build @@ -9,7 +9,7 @@ trigger-builder-pack-builder-build() { declare trigger="builder-build" declare BUILDER_TYPE="$1" APP="$2" SOURCECODE_WORK_DIR="$3" - if [[ "$BUILDER_TYPE" != "cnb" ]]; then + if [[ "$BUILDER_TYPE" != "pack" ]]; then return fi @@ -33,13 +33,13 @@ trigger-builder-pack-builder-build() { config_export app "$APP" --format envfile --merged >"$TMP_FILE" - plugn trigger pre-build-cnb "$APP" "$SOURCECODE_WORK_DIR" + plugn trigger pre-build-pack "$APP" "$SOURCECODE_WORK_DIR" pack build "$IMAGE" --builder "$DOKKU_CNB_BUILDER" --path "$SOURCECODE_WORK_DIR" --default-process web docker-image-labeler --label=com.dokku.image-stage=build --label=com.dokku.app-name=$APP --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=dokku "$IMAGE" - plugn trigger post-build-cnb "$APP" "$SOURCECODE_WORK_DIR" + plugn trigger post-build-pack "$APP" "$SOURCECODE_WORK_DIR" } trigger-builder-pack-builder-build "$@" diff --git a/plugins/builder-pack/builder-detect b/plugins/builder-pack/builder-detect index b40d1d52ffc..d269c11dcd3 100755 --- a/plugins/builder-pack/builder-detect +++ b/plugins/builder-pack/builder-detect @@ -9,12 +9,12 @@ trigger-builder-pack-builder-detect() { declare APP="$1" SOURCECODE_WORK_DIR="$2" if [[ -f "$SOURCECODE_WORK_DIR/project.toml" ]]; then - echo "cnb" + echo "pack" return fi if [[ "$(config_get "$APP" DOKKU_CNB_EXPERIMENTAL || true)" == "1" ]]; then - echo "cnb" + echo "pack" return fi } diff --git a/plugins/builder-pack/builder-release b/plugins/builder-pack/builder-release index 95e02836951..a5819abd052 100755 --- a/plugins/builder-pack/builder-release +++ b/plugins/builder-pack/builder-release @@ -8,15 +8,15 @@ trigger-builder-pack-builder-release() { declare trigger="builder-release" declare BUILDER_TYPE="$1" APP="$2" IMAGE_TAG="$3" - if [[ "$BUILDER_TYPE" != "cnb" ]]; then + if [[ "$BUILDER_TYPE" != "pack" ]]; then return fi - plugn trigger pre-release-cnb "$APP" "$IMAGE_TAG" + plugn trigger pre-release-pack "$APP" "$IMAGE_TAG" local IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG") docker-image-labeler --label=com.dokku.image-stage=release --label=com.dokku.app-name=$APP --label=org.label-schema.schema-version=1.0 --label=org.label-schema.vendor=dokku --label=dokku "$IMAGE" - plugn trigger post-release-cnb "$APP" "$IMAGE_TAG" + plugn trigger post-release-pack "$APP" "$IMAGE_TAG" } trigger-builder-pack-builder-release "$@" diff --git a/plugins/common/functions b/plugins/common/functions index a5b5a371114..df6ed38c65e 100755 --- a/plugins/common/functions +++ b/plugins/common/functions @@ -639,7 +639,7 @@ dokku_release() { local IMAGE=$(get_app_image_name "$APP" "$IMAGE_TAG") if is_image_cnb_based "$IMAGE"; then - IMAGE_SOURCE_TYPE="cnb" + IMAGE_SOURCE_TYPE="pack" fi plugn trigger builder-release "$IMAGE_SOURCE_TYPE" "$APP" "$IMAGE_TAG" @@ -710,7 +710,7 @@ dokku_receive() { local DOKKU_CNB_EXPERIMENTAL="$(config_get "$APP" DOKKU_CNB_EXPERIMENTAL || true)" if [[ "$DOKKU_CNB_EXPERIMENTAL" == "1" ]]; then - IMAGE_SOURCE_TYPE="cnb" + IMAGE_SOURCE_TYPE="pack" fi DOKKU_QUIET_OUTPUT=1 config_set --no-restart "$APP" DOKKU_APP_TYPE="$IMAGE_SOURCE_TYPE" diff --git a/plugins/git/functions b/plugins/git/functions index a67b795e1b8..16ea953a7bd 100755 --- a/plugins/git/functions +++ b/plugins/git/functions @@ -53,7 +53,7 @@ git_trigger_build() { plugn trigger post-extract "$APP" "$TMP_WORK_DIR" "$REV" - BUILDER="$(plugn trigger builder-detect "$APP" "$TMP_WORK_DIR" | tail -n1 || true)" + BUILDER="$(plugn trigger builder-detect "$APP" "$TMP_WORK_DIR" | head -n1 || true)" [[ -z "$BUILDER" ]] && BUILDER="herokuish" plugn trigger pre-receive-app "$APP" "$BUILDER" "$TMP_WORK_DIR" "$REV" diff --git a/plugins/scheduler-docker-local/scheduler-deploy b/plugins/scheduler-docker-local/scheduler-deploy index d7a1696959e..dd19ef6b76f 100755 --- a/plugins/scheduler-docker-local/scheduler-deploy +++ b/plugins/scheduler-docker-local/scheduler-deploy @@ -28,7 +28,7 @@ trigger-scheduler-docker-local-scheduler-deploy() { is_image_herokuish_based "$IMAGE" "$APP" && DOKKU_HEROKUISH=true local IMAGE_SOURCE_TYPE="dockerfile" [[ "$DOKKU_HEROKUISH" == "true" ]] && IMAGE_SOURCE_TYPE="herokuish" - [[ "$DOKKU_CNB" == "true" ]] && IMAGE_SOURCE_TYPE="cnb" + [[ "$DOKKU_CNB" == "true" ]] && IMAGE_SOURCE_TYPE="pack" local DOKKU_SCALE_FILE="$DOKKU_ROOT/$APP/DOKKU_SCALE" local oldids=$(get_app_container_ids "$APP") diff --git a/plugins/tar/functions b/plugins/tar/functions index c95d2930336..fc7e6698d42 100755 --- a/plugins/tar/functions +++ b/plugins/tar/functions @@ -45,7 +45,7 @@ tar_trigger_build() { plugn trigger post-extract "$APP" "$TMP_WORK_DIR" "$REV" - BUILDER="$(plugn trigger builder-detect "$APP" "$TMP_WORK_DIR" | tail -n1 || true)" + BUILDER="$(plugn trigger builder-detect "$APP" "$TMP_WORK_DIR" | head -n1 || true)" [[ -z "$BUILDER" ]] && BUILDER="herokuish" plugn trigger pre-receive-app "$APP" "$BUILDER" "$TMP_WORK_DIR" "$REV" diff --git a/tests/unit/builder.bats b/tests/unit/builder.bats index d6f620c4ef1..f978bb73dd0 100644 --- a/tests/unit/builder.bats +++ b/tests/unit/builder.bats @@ -10,7 +10,7 @@ teardown() { destroy_app } -@test "(builder) builder-detect [cnb]" { +@test "(builder) builder-detect [pack]" { local TMP=$(mktemp -d "/tmp/dokku.me.XXXXX") trap 'popd &>/dev/null || true; rm -rf "$TMP"' INT TERM @@ -21,11 +21,11 @@ teardown() { assert_success chown -R dokku:dokku "$TMP" - run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | tail -n1" + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | head -n1" echo "output: $output" echo "status: $status" assert_success - assert_output "cnb" + assert_output "pack" sudo rm -rf $TMP/* echo "output: $output" @@ -39,11 +39,11 @@ teardown() { assert_success chown -R dokku:dokku "$TMP" - run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | tail -n1" + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | head -n1" echo "output: $output" echo "status: $status" assert_success - assert_output "cnb" + assert_output "pack" } @test "(builder) builder-detect [dockerfile]" { @@ -56,7 +56,7 @@ teardown() { assert_success chown -R dokku:dokku "$TMP" - run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | tail -n1" + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | head -n1" echo "output: $output" echo "status: $status" assert_success @@ -77,7 +77,7 @@ teardown() { assert_success chown -R dokku:dokku "$TMP" - run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | tail -n1" + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | head -n1" echo "output: $output" echo "status: $status" assert_success @@ -95,7 +95,7 @@ teardown() { assert_success chown -R dokku:dokku "$TMP" - run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | tail -n1" + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | head -n1" echo "output: $output" echo "status: $status" assert_success @@ -113,7 +113,7 @@ teardown() { assert_success chown -R dokku:dokku "$TMP" - run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | tail -n1" + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | head -n1" echo "output: $output" echo "status: $status" assert_success From 8c34cfe64e45157a3a2bb82aaf6648ba539f0468 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 28 Feb 2021 17:25:03 -0500 Subject: [PATCH 5/7] feat: implement builder plugin This plugin will allow users to override the builder used for their application, enabling users to use custom builders if desired. --- docs/appendices/0.24.0-migration-guide.md | 2 +- .../deployment/builders/builder-management.md | 108 ++++++++++++++++++ .../cloud-native-buildpacks.md | 22 +++- .../{methods => builders}/dockerfiles.md | 24 ++-- .../herokuish-buildpacks.md | 17 +++ docs/template.html | 11 +- docs/viewdocs.json | 9 ++ plugins/builder/.gitignore | 9 ++ plugins/builder/Makefile | 6 + plugins/builder/builder-detect | 1 + plugins/builder/builder.go | 13 +++ plugins/builder/go.mod | 12 ++ plugins/builder/go.sum | 10 ++ plugins/builder/plugin.toml | 4 + plugins/builder/report.go | 44 +++++++ plugins/builder/src/commands/commands.go | 56 +++++++++ .../builder/src/subcommands/subcommands.go | 50 ++++++++ plugins/builder/src/triggers/triggers.go | 39 +++++++ plugins/builder/subcommands.go | 29 +++++ plugins/builder/triggers.go | 36 ++++++ tests/unit/builder.bats | 92 +++++++++++++-- 21 files changed, 566 insertions(+), 28 deletions(-) create mode 100644 docs/deployment/builders/builder-management.md rename docs/deployment/{methods => builders}/cloud-native-buildpacks.md (78%) rename docs/deployment/{methods => builders}/dockerfiles.md (86%) rename docs/deployment/{methods => builders}/herokuish-buildpacks.md (92%) create mode 100644 plugins/builder/.gitignore create mode 100644 plugins/builder/Makefile create mode 120000 plugins/builder/builder-detect create mode 100644 plugins/builder/builder.go create mode 100644 plugins/builder/go.mod create mode 100644 plugins/builder/go.sum create mode 100644 plugins/builder/plugin.toml create mode 100644 plugins/builder/report.go create mode 100644 plugins/builder/src/commands/commands.go create mode 100644 plugins/builder/src/subcommands/subcommands.go create mode 100644 plugins/builder/src/triggers/triggers.go create mode 100644 plugins/builder/subcommands.go create mode 100644 plugins/builder/triggers.go diff --git a/docs/appendices/0.24.0-migration-guide.md b/docs/appendices/0.24.0-migration-guide.md index fb104995d91..4815728f80f 100644 --- a/docs/appendices/0.24.0-migration-guide.md +++ b/docs/appendices/0.24.0-migration-guide.md @@ -7,4 +7,4 @@ ## Deprecations -- The 1.0.0 release of Dokku will no longer select buildpack builders over dockerfile builders if both builders match. Instead, Dokku will choose the first builder that responds to the `builder-detect` trigger. +- The 1.0.0 release of Dokku will no longer select buildpack builders over dockerfile builders if both builders match. Instead, Dokku will choose the first builder that responds to the `builder-detect` trigger. Users that wish to use a specific builder may set a builder using the `builder:set` command, which will force Dokku to use the specified builder regardless of what might be auto-detected. diff --git a/docs/deployment/builders/builder-management.md b/docs/deployment/builders/builder-management.md new file mode 100644 index 00000000000..72824b11dc1 --- /dev/null +++ b/docs/deployment/builders/builder-management.md @@ -0,0 +1,108 @@ +# Builder Management + +> New as of 0.24.0 + +``` +builder:report [] [] # Displays a builder report for one or more apps +builder:set () # Set or clear a builder property for an app +``` + +Builders are a way of customizing how an app is built from a source, allowing users flexibility in how artifacts are created for later scheduling. + +## Usage + +### Builder selection + +Dokku supports the following built-in builders: + +- `builder-dockerfile`: Builds apps using a `Dockerfile` via `docker build`. See the [dockerfile builder documentation](/docs/deployment/builders/dockerfiles.md) for more information on how this builder functions. +- `builder-herokuish`: Builds apps with Heroku's v2a Buildpack specification via `gliderlabs/herokuish`. See the [herokuish builder documentation](/docs/deployment/builders/herokuish-buildpacks.md) for more information on how this builder functions. +- `builder-pack`: Builds apps with Cloud Native Buildpacks via the `pack-cli` tool. See the [cloud native buildpacks builder documentation](/docs/deployment/builders/cloud-native-buildpacks.md) for more information on how this builder functions. + +Builders run a detection script against a source code repository, and the first detected builder will be used to build the app artifact. The exception to this is when a `Dockerfile` is detected and the app is also able to use either `herokuish` or `pack-cli` for building, in which case one of the latter will be chosen. + +### Overriding the auto-selected builder + +If desired, the builder can be specified via the `builder:set` command by speifying a value for `selected`. The selected builder will always be used. + +```shell +dokku builder:set node-js-app selected dockerfile +``` + +The default value may be set by passing an empty value for the option: + +```shell +dokku builder:set node-js-app selected +``` + +The `selected` property can also be set globally. The global default is an empty string, and auto-detection will be performed when no value is set per-app or globally. + +```shell +dokku builder:set --global selected herokuish +``` + +The default value may be set by passing an empty value for the option. + +```shell +dokku builder:set --global selected +``` + +### Displaying builder reports for an app + +You can get a report about the app's builder status using the `builder:report` command: + +```shell +dokku builder:report +``` + +``` +=====> node-js-app builder information + Builder computed selected: herokuish + Builder global selected: herokuish + Builder selected: herokuish +=====> python-sample builder information + Builder computed selected: dockerfile + Builder global selected: herokuish + Builder selected: dockerfile +=====> ruby-sample builder information + Builder computed selected: herokuish + Builder global selected: herokuish + Builder selected: +``` + +You can run the command for a specific app also. + +```shell +dokku builder:report node-js-app +``` + +``` +=====> node-js-app builder information + Builder selected: herokuish +``` + +You can pass flags which will output only the value of the specific information you want. For example: + +```shell +dokku builder:report node-js-app --builder-selected +``` + +### Custom builders + +To create a custom builder, the following triggers must be implemented: + +- `builder-build`: + - arguments: `BUILDER_TYPE` `APP` `SOURCECODE_WORK_DIR` + - description: Creates a docker image named with the output of `common#get_app_image_name $APP`. +- `builder-detect`: + - arguments: `APP` `SOURCECODE_WORK_DIR` + - description: Outputs the name of the builder (without the `builder-` prefix) to use to build the app. +- `builder-release`: + - arguments: `BUILDER_TYPE` `APP` `IMAGE_AG` + - description: A post-build, pre-release trigger that can be used to post-process the image. Usually simply tags and labels the image appropriately. + +Custom plugins names _must_ have the prefix `builder-` or builder overriding via `builder:set` may not function as expected. + +Builders can use any tools available on the system to build the docker image, and may even be used to schedule building off-server. The only current requirement is that the image must exist on the server at the end of the `builder-build` command, though this requirement may be relaxed in a future release. + +For a simple example of how to implement this trigger, see `builder-pack`, which utilizes a cli tool - `pack-cli` - to generate an OCI image that is compatible with Docker and can be scheduled by the official scheduling plugins. diff --git a/docs/deployment/methods/cloud-native-buildpacks.md b/docs/deployment/builders/cloud-native-buildpacks.md similarity index 78% rename from docs/deployment/methods/cloud-native-buildpacks.md rename to docs/deployment/builders/cloud-native-buildpacks.md index ce1588b456a..95df0b51a35 100644 --- a/docs/deployment/methods/cloud-native-buildpacks.md +++ b/docs/deployment/builders/cloud-native-buildpacks.md @@ -5,23 +5,39 @@ ``` buildpacks:set-property [--global|] # Set or clear a buildpacks property for an app ``` + Cloud Native Buildpacks are an evolution over the Buildpacks technology provided by the Herokuish builder. See the [herokuish buildpacks documentation](/docs/deployment/methods/herokuish-buildpacks.md) for more information on how to clear buildpack build cache for an application. > Warning: This functionality uses the `pack` cli from the [Cloud Native Buildpacks](https://buildpacks.io) project to build apps. As the integration is experimental in Dokku, it is likely to change over time. ## Usage -To use this builder instead of either `Dockerfile` or `herokuish`, you must set the `DOKKU_CNB_EXPERIMENTAL` environment variable for your app to `1`. +### Detection + +This builder will be auto-detected in either the following cases: + +- The `DOKKU_CNB_EXPERIMENTAL` app environment variable is set to `1`. + ```shell + dokku config:set --no-restart node-js-app DOKKU_CNB_EXPERIMENTAL=1 + ``` +- A `.project.toml` file exists in the root of the app repository. + - This file is consumed by `pack-cli` and used to describe how the app is built. + +The builder can also be specified via the `builder:set` command: ```shell -dokku config:set --no-restart node-js-app DOKKU_CNB_EXPERIMENTAL=1 +dokku builder:set node-js-app selected pack ``` +> Dokku will only select the `dockerfile` builder if both the `herokuish` and `pack` builders are not detected and a Dockerfile exists. See the [dockerfile builder documentation](/docs/deployment/builders/dockerfiles.md) for more information on how that builder functions. + +### Requirements + The `pack` cli tool is not included by default with Dokku or as a dependency. It must also be installed as shown on [this page](https://buildpacks.io/docs/tools/pack/). Builds will proceed with the `pack` cli for the app from then on. -## Caveats +### Caveats As this functionality is highly experimental, there are a number of caveats. Please note that not all issuesare listed below. diff --git a/docs/deployment/methods/dockerfiles.md b/docs/deployment/builders/dockerfiles.md similarity index 86% rename from docs/deployment/methods/dockerfiles.md rename to docs/deployment/builders/dockerfiles.md index 0a746534569..581fb41e222 100644 --- a/docs/deployment/methods/dockerfiles.md +++ b/docs/deployment/builders/dockerfiles.md @@ -6,12 +6,20 @@ While Dokku normally defaults to using [Heroku buildpacks](https://devcenter.her > Dockerfile support is considered a *power user* feature. By using Dockerfile-based deployment, you agree that you will not have the same comfort as that enjoyed by buildpack users, and Dokku features may work differently. Differences between the two systems will be documented here. -To use a Dockerfile for deployment, commit a valid `Dockerfile` to the root of your repository and push the repository to your Dokku installation. If this file is detected, Dokku will default to using it to construct containers *except* in the following two cases: +## Usage -- The application has a `BUILDPACK_URL` environment variable set via the `dokku config:set` command or in a committed `.env` file. In this case, Dokku will use your specified buildpack. -- The application has a `.buildpacks` file in the root of the repository. In this case, Dokku will use your specified buildpack(s). +### Detection -## Switching from buildpack deployments +This builder will be auto-detected in the following case: + +- A `Dockerfile` exists in the root of the app repository. + +Dokku will only select the `dockerfile` builder if both the `herokuish` and `pack` builders are not detected and a Dockerfile exists. For more information on how those are detected, see the following links: + +- [Cloud Native Buildpacks documentation](/docs/deployment/builders/cloud-native-buildpacks.md#detection) +- [Herokuish documentation](/docs/deployment/builders/herokuish-buildpacks.md#detection) + +### Switching from buildpack deployments If an application was previously deployed via buildpacks, the following commands should be run before a Dockerfile deploy will succeed: @@ -19,7 +27,7 @@ If an application was previously deployed via buildpacks, the following commands dokku config:unset --no-restart node-js-app DOKKU_PROXY_PORT_MAP ``` -## Build-time configuration variables +### Build-time configuration variables For security reasons - and as per [Docker recommendations](https://github.com/docker/docker/issues/13490) - Dockerfile-based deploys have variables available only during runtime. @@ -65,11 +73,11 @@ ENV NODE_ENV ${NODE_ENV} RUN echo $NODE_ENV ``` -## Building images with Docker Buildkit +### Building images with Docker Buildkit If your Dockerfile is using Docker engine's [buildkit](https://docs.docker.com/develop/develop-images/build_enhancements/) (not to be confused with buildpacks), then the `DOCKER_BUILDKIT=1` environment variable needs to be set. One way to do this is to edit `/etc/environment` on your dokku host and reboot your instance. Note, for complete build log output, you should also set `BUILDKIT_PROGRESS=plain` in the same file. -## Customizing the run command +### Customizing the run command By default no arguments are passed to `docker run` when deploying the container and the `CMD` or `ENTRYPOINT` defined in the `Dockerfile` are executed. You can take advantage of docker ability of overriding the `CMD` or passing parameters to your `ENTRYPOINT` setting `$DOKKU_DOCKERFILE_START_CMD`. Let's say for example you are deploying a base Node.js image, with the following `ENTRYPOINT`: @@ -118,6 +126,6 @@ started by running `docker run bin/run-worker.sh` (the actual `docker run` comma complex, but this is the basic idea). If you use an `ENTRYPOINT` in your `Dockerfile`, the lines in your `Procfile` will be passed as arguments to the `ENTRYPOINT` script instead of being executed. -## Exposed ports +### Exposed ports See the [port management documentation](/docs/networking/port-management.md) for more information on how Dokku exposes ports for applications and how you can configure these for your app. diff --git a/docs/deployment/methods/herokuish-buildpacks.md b/docs/deployment/builders/herokuish-buildpacks.md similarity index 92% rename from docs/deployment/methods/herokuish-buildpacks.md rename to docs/deployment/builders/herokuish-buildpacks.md index 3803d536b1c..c6f40dfff6b 100644 --- a/docs/deployment/methods/herokuish-buildpacks.md +++ b/docs/deployment/builders/herokuish-buildpacks.md @@ -25,6 +25,23 @@ This page will cover usage of the `buildpacks` plugin. ## Usage +### Detection + +This builder will be auto-detected in either the following cases: + +- The `BUILDPACK_URL` app environment variable is set. + - This can be done via `dokku config:set` or via a committed `.env` file in the root of the repository. See the [environment variable documentation](/docs/configuration/environment-variables.md) for more details. +- A `.buildpacks` file exists in the root of the app repository. + - This can be via a committed `.buildpacks` file or managed via the `buildpacks` plugin commands. + +The builder can also be specified via the `builder:set` command: + +```shell +dokku builder:set node-js-app selected herokuish +``` + +> Dokku will only select the `dockerfile` builder if both the `herokuish` and `pack` builders are not detected and a Dockerfile exists. See the [dockerfile builder documentation](/docs/deployment/builders/dockerfiles.md) for more information on how that builder functions. + ### Listing Buildpacks in Use The `buildpacks:list` command can be used to show buildpacks that have been set for an app. This will omit any auto-detected buildpacks. diff --git a/docs/template.html b/docs/template.html index 24c9f4c25de..f5522334e31 100644 --- a/docs/template.html +++ b/docs/template.html @@ -127,11 +127,14 @@

User Management Zero Downtime Deploy Checks - Deployment Methods + Builders + + Builder Management + Cloud Native Buildpacks + Herokuish Buildpacks + Dockerfile Deployment - Cloud Native Buildpacks - Herokuish Buildpacks - Dockerfile Deployment + Deployment Methods Docker Image Deployment Git Deployment Tarfile Deployment diff --git a/docs/viewdocs.json b/docs/viewdocs.json index 6cacc72cc2d..7d13a1828c0 100644 --- a/docs/viewdocs.json +++ b/docs/viewdocs.json @@ -16,6 +16,15 @@ "deployment/buildpacks": "deployment/methods/herokuish-buildpacks/", "deployment/methods/buildpacks": "deployment/methods/herokuish-buildpacks/", "deployment/dockerfiles": "deployment/methods/dockerfiles/", + + "deployment/buildpacks": "deployment/builders/herokuish-buildpacks/", + "deployment/methods/buildpacks": "deployment/builders/herokuish-buildpacks/", + "deployment/dockerfiles": "deployment/builders/dockerfiles/", + + "deployment/methods/cloud-native-buildpacks": "deployment/builders/cloud-native-buildpacks/", + "deployment/methods/dockerfiles": "deployment/builders/dockerfiles/", + "deployment/methods/herokuish-buildpacks": "deployment/builders/herokuish-buildpacks/", + "deployment/images": "deployment/methods/images/", "configuration-management": "configuration/environment-variables/", "deployment/ssl-configuration": "configuration/ssl/", diff --git a/plugins/builder/.gitignore b/plugins/builder/.gitignore new file mode 100644 index 00000000000..dee017e8f3c --- /dev/null +++ b/plugins/builder/.gitignore @@ -0,0 +1,9 @@ +/commands +/core-post-deploy +/subcommands/* +/triggers/* +/triggers +/network-* +/install +/post-* +/report diff --git a/plugins/builder/Makefile b/plugins/builder/Makefile new file mode 100644 index 00000000000..8dce99ce845 --- /dev/null +++ b/plugins/builder/Makefile @@ -0,0 +1,6 @@ +SUBCOMMANDS = subcommands/report subcommands/set +TRIGGERS = triggers/builder-detect triggers/install triggers/post-delete triggers/report +BUILD = commands subcommands triggers +PLUGIN_NAME = builder + +include ../../common.mk diff --git a/plugins/builder/builder-detect b/plugins/builder/builder-detect new file mode 120000 index 00000000000..fa05d2b3eb6 --- /dev/null +++ b/plugins/builder/builder-detect @@ -0,0 +1 @@ +triggers \ No newline at end of file diff --git a/plugins/builder/builder.go b/plugins/builder/builder.go new file mode 100644 index 00000000000..22ef608b4a5 --- /dev/null +++ b/plugins/builder/builder.go @@ -0,0 +1,13 @@ +package builder + +var ( + // DefaultProperties is a map of all valid network properties with corresponding default property values + DefaultProperties = map[string]string{ + "selected": "", + } + + // GlobalProperties is a map of all valid global network properties + GlobalProperties = map[string]bool{ + "selected": true, + } +) diff --git a/plugins/builder/go.mod b/plugins/builder/go.mod new file mode 100644 index 00000000000..a382f3f9c71 --- /dev/null +++ b/plugins/builder/go.mod @@ -0,0 +1,12 @@ +module github.com/dokku/dokku/plugins/builder + +go 1.15 + +require ( + github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 + github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 + github.com/dokku/dokku/plugins/common v0.0.0-00010101000000-000000000000 + github.com/spf13/pflag v1.0.5 // indirect +) + +replace github.com/dokku/dokku/plugins/common => ../common diff --git a/plugins/builder/go.sum b/plugins/builder/go.sum new file mode 100644 index 00000000000..58c90470707 --- /dev/null +++ b/plugins/builder/go.sum @@ -0,0 +1,10 @@ +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= +github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27 h1:HHUr4P/aKh4quafGxDT9LDasjGdlGkzLbfmmrlng3kA= +github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27/go.mod h1:VQx0hjo2oUeQkQUET7wRwradO6f+fN5jzXgB/zROxxE= +github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3 h1:utdYOikI1XjNtTFGCwSM6OmFJblU4ld4gACoJsbadJg= +github.com/ryanuber/columnize v1.1.2-0.20190319233515-9e6335e58db3/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/plugins/builder/plugin.toml b/plugins/builder/plugin.toml new file mode 100644 index 00000000000..138b6348ad6 --- /dev/null +++ b/plugins/builder/plugin.toml @@ -0,0 +1,4 @@ +[plugin] +description = "dokku core builder plugin" +version = "0.23.9" +[plugin.config] diff --git a/plugins/builder/report.go b/plugins/builder/report.go new file mode 100644 index 00000000000..57eaf46046d --- /dev/null +++ b/plugins/builder/report.go @@ -0,0 +1,44 @@ +package builder + +import ( + "github.com/dokku/dokku/plugins/common" +) + +// ReportSingleApp is an internal function that displays the builder report for one or more apps +func ReportSingleApp(appName string, format string, infoFlag string) error { + if err := common.VerifyAppName(appName); err != nil { + return err + } + + flags := map[string]common.ReportFunc{ + "--builder-computed-selected": reportComputedSelected, + "--builder-global-selected": reportGlobalSelected, + "--builder-selected": reportSelected, + } + + flagKeys := []string{} + for flagKey := range flags { + flagKeys = append(flagKeys, flagKey) + } + + trimPrefix := false + uppercaseFirstCharacter := true + infoFlags := common.CollectReport(appName, infoFlag, flags) + return common.ReportSingleApp("builder", appName, infoFlag, infoFlags, flagKeys, format, trimPrefix, uppercaseFirstCharacter) +} + +func reportComputedSelected(appName string) string { + value := reportSelected(appName) + if value == "" { + value = reportGlobalSelected(appName) + } + + return value +} +func reportGlobalSelected(appName string) string { + return common.PropertyGet("builder", "--global", "selected") +} + +func reportSelected(appName string) string { + return common.PropertyGet("builder", appName, "selected") +} diff --git a/plugins/builder/src/commands/commands.go b/plugins/builder/src/commands/commands.go new file mode 100644 index 00000000000..55169ef4235 --- /dev/null +++ b/plugins/builder/src/commands/commands.go @@ -0,0 +1,56 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strconv" + "strings" + + "github.com/dokku/dokku/plugins/common" +) + +const ( + helpHeader = `Usage: dokku builder[:COMMAND] + +Manage builder settings for an app + +Additional commands:` + + helpContent = ` + builder:report [] [], Displays a builder report for one or more apps + builder:set (), Set or clear a builder property for an app +` +) + +func main() { + flag.Usage = usage + flag.Parse() + + cmd := flag.Arg(0) + switch cmd { + case "builder", "builder:help": + usage() + case "help": + command := common.NewShellCmd(fmt.Sprintf("ps -o command= %d", os.Getppid())) + command.ShowOutput = false + output, err := command.Output() + + if err == nil && strings.Contains(string(output), "--all") { + fmt.Println(helpContent) + } else { + fmt.Print("\n builder, Manage builder settings for an app\n") + } + default: + dokkuNotImplementExitCode, err := strconv.Atoi(os.Getenv("DOKKU_NOT_IMPLEMENTED_EXIT")) + if err != nil { + fmt.Println("failed to retrieve DOKKU_NOT_IMPLEMENTED_EXIT environment variable") + dokkuNotImplementExitCode = 10 + } + os.Exit(dokkuNotImplementExitCode) + } +} + +func usage() { + common.CommandUsage(helpHeader, helpContent) +} diff --git a/plugins/builder/src/subcommands/subcommands.go b/plugins/builder/src/subcommands/subcommands.go new file mode 100644 index 00000000000..e6d69269182 --- /dev/null +++ b/plugins/builder/src/subcommands/subcommands.go @@ -0,0 +1,50 @@ +package main + +import ( + "fmt" + "os" + "strings" + + "github.com/dokku/dokku/plugins/builder" + "github.com/dokku/dokku/plugins/common" + + flag "github.com/spf13/pflag" +) + +// main entrypoint to all subcommands +func main() { + parts := strings.Split(os.Args[0], "/") + subcommand := parts[len(parts)-1] + + var err error + switch subcommand { + case "report": + args := flag.NewFlagSet("builder:report", flag.ExitOnError) + format := args.String("format", "stdout", "format: [ stdout | json ]") + osArgs, infoFlag, flagErr := common.ParseReportArgs("builder", os.Args[2:]) + if flagErr == nil { + args.Parse(osArgs) + appName := args.Arg(0) + err = builder.CommandReport(appName, *format, infoFlag) + } + case "set": + args := flag.NewFlagSet("builder:set", flag.ExitOnError) + global := args.Bool("global", false, "--global: set a global property") + args.Parse(os.Args[2:]) + appName := args.Arg(0) + property := args.Arg(1) + value := args.Arg(2) + if *global { + appName = "--global" + property = args.Arg(0) + value = args.Arg(1) + } + err = builder.CommandSet(appName, property, value) + default: + err = fmt.Errorf("Invalid plugin subcommand call: %s", subcommand) + } + + if err != nil { + common.LogFailWithError(err) + } +} diff --git a/plugins/builder/src/triggers/triggers.go b/plugins/builder/src/triggers/triggers.go new file mode 100644 index 00000000000..ff3e4beb25e --- /dev/null +++ b/plugins/builder/src/triggers/triggers.go @@ -0,0 +1,39 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strings" + + "github.com/dokku/dokku/plugins/builder" + "github.com/dokku/dokku/plugins/common" +) + +// main entrypoint to all triggers +func main() { + parts := strings.Split(os.Args[0], "/") + trigger := parts[len(parts)-1] + flag.Parse() + + var err error + switch trigger { + case "builder-detect": + appName := flag.Arg(0) + err = builder.TriggerBuilderDetect(appName) + case "install": + err = builder.TriggerInstall() + case "post-delete": + appName := flag.Arg(0) + err = builder.TriggerPostDelete(appName) + case "report": + appName := flag.Arg(0) + err = builder.ReportSingleApp(appName, "", "") + default: + err = fmt.Errorf("Invalid plugin trigger call: %s", trigger) + } + + if err != nil { + common.LogFailWithError(err) + } +} diff --git a/plugins/builder/subcommands.go b/plugins/builder/subcommands.go new file mode 100644 index 00000000000..18e04ac0e1d --- /dev/null +++ b/plugins/builder/subcommands.go @@ -0,0 +1,29 @@ +package builder + +import ( + "github.com/dokku/dokku/plugins/common" +) + +// CommandReport displays a network report for one or more apps +func CommandReport(appName string, format string, infoFlag string) error { + if len(appName) == 0 { + apps, err := common.DokkuApps() + if err != nil { + return err + } + for _, appName := range apps { + if err := ReportSingleApp(appName, format, infoFlag); err != nil { + return err + } + } + return nil + } + + return ReportSingleApp(appName, format, infoFlag) +} + +// CommandSet set or clear a builder property for an app +func CommandSet(appName string, property string, value string) error { + common.CommandPropertySet("builder", appName, property, value, DefaultProperties, GlobalProperties) + return nil +} diff --git a/plugins/builder/triggers.go b/plugins/builder/triggers.go new file mode 100644 index 00000000000..0cfed983777 --- /dev/null +++ b/plugins/builder/triggers.go @@ -0,0 +1,36 @@ +package builder + +import ( + "fmt" + + "github.com/dokku/dokku/plugins/common" +) + +// TriggerBuilderDetect outputs a manually selected builder for the app +func TriggerBuilderDetect(appName string) error { + if builder := common.PropertyGet("builder", appName, "selected"); builder != "" { + fmt.Println(builder) + return nil + } + + if builder := common.PropertyGet("builder", "--global", "selected"); builder != "" { + fmt.Println(builder) + return nil + } + + return nil +} + +// TriggerInstall runs the install step for the builder plugin +func TriggerInstall() error { + if err := common.PropertySetup("builder"); err != nil { + return fmt.Errorf("Unable to install the builder plugin: %s", err.Error()) + } + + return nil +} + +// TriggerPostDelete destroys the builder property for a given app container +func TriggerPostDelete(appName string) error { + return common.PropertyDestroy("builder", appName) +} diff --git a/tests/unit/builder.bats b/tests/unit/builder.bats index f978bb73dd0..3ee605314d5 100644 --- a/tests/unit/builder.bats +++ b/tests/unit/builder.bats @@ -10,6 +10,74 @@ teardown() { destroy_app } +@test "(builder) builder-detect [set]" { + local TMP=$(mktemp -d "/tmp/dokku.me.XXXXX") + trap 'popd &>/dev/null || true; rm -rf "$TMP"' INT TERM + + # test project.toml + run touch "$TMP/project.toml" + echo "output: $output" + echo "status: $status" + assert_success + + chown -R dokku:dokku "$TMP" + + run /bin/bash -c "dokku builder:set --global selected pack" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP" + echo "output: $output" + echo "status: $status" + assert_success + assert_line 0 "pack" + + run /bin/bash -c "dokku builder:set $TEST_APP selected pack" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP" + echo "output: $output" + echo "status: $status" + assert_success + assert_line 0 "pack" + + run /bin/bash -c "dokku builder:set --global selected other" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP" + echo "output: $output" + echo "status: $status" + assert_success + assert_line 0 "pack" + + run /bin/bash -c "dokku builder:set $TEST_APP selected" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP" + echo "output: $output" + echo "status: $status" + assert_success + assert_line 0 "other" + + run /bin/bash -c "dokku builder:set --global selected" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP" + echo "output: $output" + echo "status: $status" + assert_success + assert_line 0 "pack" +} + @test "(builder) builder-detect [pack]" { local TMP=$(mktemp -d "/tmp/dokku.me.XXXXX") trap 'popd &>/dev/null || true; rm -rf "$TMP"' INT TERM @@ -21,11 +89,11 @@ teardown() { assert_success chown -R dokku:dokku "$TMP" - run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | head -n1" + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP" echo "output: $output" echo "status: $status" assert_success - assert_output "pack" + assert_line 0 "pack" sudo rm -rf $TMP/* echo "output: $output" @@ -39,11 +107,11 @@ teardown() { assert_success chown -R dokku:dokku "$TMP" - run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | head -n1" + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP" echo "output: $output" echo "status: $status" assert_success - assert_output "pack" + assert_line 0 "pack" } @test "(builder) builder-detect [dockerfile]" { @@ -56,11 +124,11 @@ teardown() { assert_success chown -R dokku:dokku "$TMP" - run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | head -n1" + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP" echo "output: $output" echo "status: $status" assert_success - assert_output "dockerfile" + assert_line 0 "dockerfile" } @test "(builder) builder-detect [herokuish]" { @@ -77,11 +145,11 @@ teardown() { assert_success chown -R dokku:dokku "$TMP" - run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | head -n1" + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP" echo "output: $output" echo "status: $status" assert_success - assert_output "herokuish" + assert_line 0 "herokuish" sudo rm -rf $TMP/* echo "output: $output" @@ -95,11 +163,11 @@ teardown() { assert_success chown -R dokku:dokku "$TMP" - run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | head -n1" + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP" echo "output: $output" echo "status: $status" assert_success - assert_output "herokuish" + assert_line 0 "herokuish" sudo rm -rf $TMP/* echo "output: $output" @@ -113,9 +181,9 @@ teardown() { assert_success chown -R dokku:dokku "$TMP" - run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP | head -n1" + run /bin/bash -c "dokku plugin:trigger builder-detect $TEST_APP $TMP" echo "output: $output" echo "status: $status" assert_success - assert_output "herokuish" + assert_line 0 "herokuish" } From 6ea57dafe741cda9da5bcf9d2246290b8331e79a Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 28 Feb 2021 17:25:24 -0500 Subject: [PATCH 6/7] chore: add LogFailWithError to common triggers --- plugins/common/src/triggers/triggers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/common/src/triggers/triggers.go b/plugins/common/src/triggers/triggers.go index e5b342297ca..cefabbe2893 100644 --- a/plugins/common/src/triggers/triggers.go +++ b/plugins/common/src/triggers/triggers.go @@ -26,10 +26,10 @@ func main() { appName := flag.Arg(0) err = common.TriggerPostDelete(appName) default: - common.LogFail(fmt.Sprintf("Invalid plugin trigger call: %s", trigger)) + err = fmt.Errorf("Invalid plugin trigger call: %s", trigger) } if err != nil { - common.LogFail(err.Error()) + common.LogFailWithError(err) } } From 2392c69a28b618d26c2fd1958226a0a5401b81d5 Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Sun, 28 Feb 2021 17:29:34 -0500 Subject: [PATCH 7/7] docs: remove duplicate redirect entry --- docs/viewdocs.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/viewdocs.json b/docs/viewdocs.json index 7d13a1828c0..f934ccdf145 100644 --- a/docs/viewdocs.json +++ b/docs/viewdocs.json @@ -13,10 +13,6 @@ "deployment/process-management": "processes/process-management/", "deployment/one-off-processes": "processes/one-off-tasks/", - "deployment/buildpacks": "deployment/methods/herokuish-buildpacks/", - "deployment/methods/buildpacks": "deployment/methods/herokuish-buildpacks/", - "deployment/dockerfiles": "deployment/methods/dockerfiles/", - "deployment/buildpacks": "deployment/builders/herokuish-buildpacks/", "deployment/methods/buildpacks": "deployment/builders/herokuish-buildpacks/", "deployment/dockerfiles": "deployment/builders/dockerfiles/",