From 2c03567cefb42f18898de42651d4607caa788646 Mon Sep 17 00:00:00 2001 From: Paul Lietar Date: Mon, 4 Nov 2013 23:59:16 +0000 Subject: [PATCH 01/20] Add an 'add-ons' plugin, which provides heroku style add-ons. Add-ons provide access to services (e.g. database) by applications. This is still WIP. --- plugins/addons/commands | 114 +++++++++++++++++++++++++++++++++++++ plugins/addons/pre-delete | 3 + plugins/addons/pre-release | 40 +++++++++++++ 3 files changed, 157 insertions(+) create mode 100755 plugins/addons/commands create mode 100755 plugins/addons/pre-delete create mode 100755 plugins/addons/pre-release diff --git a/plugins/addons/commands b/plugins/addons/commands new file mode 100755 index 00000000000..dcbb57942fd --- /dev/null +++ b/plugins/addons/commands @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x + +function die +{ + echo $* >&2 + exit 1 +} + +ADDONS_ROOT=/var/lib/dokku/addons + +# Check if name is specified +if [[ $1 == addons ]] || [[ $1 == addons:* ]]; then + if [[ -z $2 ]]; then + echo "You must specify an app name" + exit 1 + else + APP="$2" + ADDONS_FILE="$DOKKU_ROOT/$APP/ADDONS" + ENV_FILE="$DOKKU_ROOT/$APP/ENV" + + # Check if app exists with the same name + if [ ! -d "$DOKKU_ROOT/$APP" ]; then + echo "App $APP does not exist" + exit 1 + fi + + [ -f $ADDONS_FILE ] || { + echo "-----> Creating $ADDONS_FILE" + touch $ADDONS_FILE + } + [ -f $ENV_FILE ] || { + echo "-----> Creating $ENV_FILE" + touch $ENV_FILE + } + fi +fi + +# +# Each line of the ADDONS file is the form "name;id;private" +# +# The combination of the id and the private data is used by the addon to generate the url +# + +case "$1" in + addons) + cut -d";" -f1 $ADDONS_FILE + ;; + addons:add) + ADDON=$3 + if [[ -z $3 ]]; then + die "You must specify an addon name" + elif [ ! -d "$ADDONS_ROOT/$ADDON" ]; then + die "Addon $ADDON does not exist" + fi + + if cut -d";" -f1 $ADDONS_FILE | grep -q "^$ADDON$"; then + die "App $APP already has addon $ADDON" + fi + + id_private=$($ADDONS_ROOT/$ADDON/provision $APP) + + echo "$ADDON;$id_private" >> $ADDONS_FILE + + ;; + addons:remove) + ADDON=$3 + if [[ -z $3 ]]; then + die "You must specify an addon name" + exit 1 + elif [ ! -d "$ADDONS_ROOT/$ADDON" ]; then + die "Addon $ADDON does not exist" + exit 1 + fi + + line=$(grep "^$ADDON;" $ADDONS_FILE) || { + die "App $APP does not have addon $ADDON" + } + + id=$(cut -d";" -f2 <<< "$line") + key=DOKKU_${ADDON^^}_URL + + $ADDONS_ROOT/$ADDON/unprovision $id + sed -i "/^$ADDON;/d" $ADDONS_FILE + sed "/^export $key=/d" $ENV_FILE | sort > $ENV_FILE + + + ;; + + addons:url) + ADDON=$3 + if [[ -z $3 ]]; then + die "You must specify an addon name" + exit 1 + elif [ ! -d "$ADDONS_ROOT/$ADDON" ]; then + die "Addon $ADDON does not exist" + exit 1 + fi + + line=$(grep "^$ADDON;" $ADDONS_FILE) || { + die "App $APP does not have addon $ADDON" + } + + id=$(cut -d";" -f2 <<< "$line") + private=$(cut -d";" -f3 <<< "$line") + + $ADDONS_ROOT/$ADDON/url $id $private + + ;; + + help) + ;; +esac + diff --git a/plugins/addons/pre-delete b/plugins/addons/pre-delete new file mode 100755 index 00000000000..4ef0a84ccc2 --- /dev/null +++ b/plugins/addons/pre-delete @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x + diff --git a/plugins/addons/pre-release b/plugins/addons/pre-release new file mode 100755 index 00000000000..1e9f18cb694 --- /dev/null +++ b/plugins/addons/pre-release @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x + +ADDONS_ROOT=/var/lib/dokku/addons + +if [[ -z $1 ]]; then + exit 1 +else + APP="$1" + ENV_FILE="$DOKKU_ROOT/$APP/ENV" + ADDONS_FILE="$DOKKU_ROOT/$APP/ADDONS" + + # Check if app exists with the same name + if [ ! -d "$DOKKU_ROOT/$APP" ]; then + exit 1 + fi + + touch $ADDONS_FILE + touch $ENV_FILE +fi + + +while read line +do + ADDON=$(cut -d";" -f1 <<< "$line") + id=$(cut -d";" -f2 <<< "$line") + private=$(cut -d";" -f3 <<< "$line") + + key=DOKKU_${ADDON^^}_URL + + if [ -d "$ADDONS_ROOT/$ADDON" ]; then + url=$($ADDONS_ROOT/$ADDON/url $id $private) + + sed "/^export $key=/d" $ENV_FILE | cat - <( echo "export $key=\"$url\"") | sort > $ENV_FILE + fi + + +done < $ADDONS_FILE + + From 41a02fab956182fe51cb5448fa8997eea2c5f5e7 Mon Sep 17 00:00:00 2001 From: Paul Lietar Date: Tue, 5 Nov 2013 20:52:45 +0000 Subject: [PATCH 02/20] Add an ADDONS.md file to describe how to use the plugin. --- ADDONS.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 ADDONS.md diff --git a/ADDONS.md b/ADDONS.md new file mode 100644 index 00000000000..7dca7e8b1ce --- /dev/null +++ b/ADDONS.md @@ -0,0 +1,36 @@ +User interface +------------------- +Four commands are available to manage plugins : +* `dokku addons` Lists all addons an application has. +* `dokku addons:add` Adds an addon to an application. This causes the addon to allocate resources for this application (eg an account, database, ..) +* `dokku addons:url` Shows the URL a plugin has given. This is only meant for debugging. In production, there's no need to read this url manually, as it is provided in an environment variable. +* `dokku addons:remove` Removes an addon from an application. This causes the addon to destroy all associated resources. + +The URL is passed to the application through the `DOKKU_${ADDON}_URL` where `${ADDON}` is the name of the add-on in uppercase. + +Add-on development +--------------------------- +Add-ons are located in `/var/lib/dokku/addons/`. Every subfolder is an add-on. +When they provision an app, add-ons should generate a unique ID, and a "private" value. +The ID is used to identify the app within the add-on. This would typically be a username or database name. +The "private" value is used to generate the URL. This would be a password for example. + +IDs and private values can take any value. They however must not include semicolons, as it is used internally by the plugin. (See the internal section) + +Add-ons must provide three executable files : +* `$ADDON/provision` This script takes one argument, the app name. It should only be used to generate understandable IDs, which contain the app name. Add-ons shouldn't use it the access the app's config files, ... +The script should output the generated id and private value on stdout, separated by a semicolon. +* `$ADDON/unprovision` This script takes one argument, the ID. It should destroy all resources associated with this ID. +* `$ADDON/url` This script takes two arguments, the ID and the private value. It should output the url on stdout. + +Add-on's are free to run the service in the way they like. They can run it on the cloud, on the host, on a single docker container or on a container per provisioned app. The plugin doesn't care, as long as the add-on provides a URL which is accessible. +Because URLs might change (docker can assign different IPs/ports after reboot), the `url` script is called each time the app is released. (BTW, this means we should release all apps at startup, rather than deploy them) + +Internals +------------ +The add-on plugin uses the `$APP/ADDONS` file to save which add-ons are in use. +Each line has the following format : + + name;id;private + +In the `pre-release` hook, the plugin generates environment variables based on the ids and private values, using the `url` script, and saves them to `$APP/ENV` From 31f39d839edd56261d7e195d55a42e2228078400 Mon Sep 17 00:00:00 2001 From: Paul Lietar Date: Tue, 5 Nov 2013 21:24:06 +0000 Subject: [PATCH 03/20] Fix indentation, refactor app/addon checking and add add-on private directory. --- ADDONS.md | 16 ++-- plugins/addons/commands | 157 ++++++++++++++++++++-------------------- 2 files changed, 87 insertions(+), 86 deletions(-) diff --git a/ADDONS.md b/ADDONS.md index 7dca7e8b1ce..1f4c4cf8c28 100644 --- a/ADDONS.md +++ b/ADDONS.md @@ -1,5 +1,4 @@ -User interface -------------------- +# User interface Four commands are available to manage plugins : * `dokku addons` Lists all addons an application has. * `dokku addons:add` Adds an addon to an application. This causes the addon to allocate resources for this application (eg an account, database, ..) @@ -8,8 +7,7 @@ Four commands are available to manage plugins : The URL is passed to the application through the `DOKKU_${ADDON}_URL` where `${ADDON}` is the name of the add-on in uppercase. -Add-on development ---------------------------- +# Add-on development Add-ons are located in `/var/lib/dokku/addons/`. Every subfolder is an add-on. When they provision an app, add-ons should generate a unique ID, and a "private" value. The ID is used to identify the app within the add-on. This would typically be a username or database name. @@ -26,8 +24,14 @@ The script should output the generated id and private value on stdout, separated Add-on's are free to run the service in the way they like. They can run it on the cloud, on the host, on a single docker container or on a container per provisioned app. The plugin doesn't care, as long as the add-on provides a URL which is accessible. Because URLs might change (docker can assign different IPs/ports after reboot), the `url` script is called each time the app is released. (BTW, this means we should release all apps at startup, rather than deploy them) -Internals ------------- +## Guidelines +### Add-ons internal files +Add-ons may want to store files for their own internal use, for eg. database storage. +Before any add-on script is ran, a `$ADDON_ROOT` environment variable is exported, which contains the path to a directory where the addon can safely save any file. The directory is guaranteed to exist before add-on scripts are ran. +Currently it is set to `$DOKKU_ROOT/.addons/$ADDON` where `$ADDON` is the addon's name. However, this might change, therefore plugins should use the `$ADDON_ROOT` variable rather than hardcoding it. + + +# Internals The add-on plugin uses the `$APP/ADDONS` file to save which add-ons are in use. Each line has the following format : diff --git a/plugins/addons/commands b/plugins/addons/commands index dcbb57942fd..30f8faf3136 100755 --- a/plugins/addons/commands +++ b/plugins/addons/commands @@ -1,28 +1,26 @@ #!/usr/bin/env bash set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x +ADDONS_PATH=/var/lib/dokku/addons + function die { - echo $* >&2 - exit 1 + echo $* >&2 + exit 1 } -ADDONS_ROOT=/var/lib/dokku/addons - -# Check if name is specified -if [[ $1 == addons ]] || [[ $1 == addons:* ]]; then - if [[ -z $2 ]]; then - echo "You must specify an app name" - exit 1 +function check_app +{ + if [[ -z $1 ]]; then + die "You must specify an app name" else - APP="$2" + APP="$1" ADDONS_FILE="$DOKKU_ROOT/$APP/ADDONS" ENV_FILE="$DOKKU_ROOT/$APP/ENV" # Check if app exists with the same name if [ ! -d "$DOKKU_ROOT/$APP" ]; then - echo "App $APP does not exist" - exit 1 + die "App $APP does not exist" fi [ -f $ADDONS_FILE ] || { @@ -34,7 +32,21 @@ if [[ $1 == addons ]] || [[ $1 == addons:* ]]; then touch $ENV_FILE } fi -fi +} + +function check_addon +{ + if [[ -z $1 ]]; then + die "You must specify an addon name" + elif [ ! -d "$ADDONS_PATH/$ADDON" ]; then + die "Addon $ADDON does not exist" + fi + ADDON=$1 + export ADDON_ROOT=$DOKKU_ROOT/.addons/$ADDON + + mkdir -p $ADDON_ROOT +} + # # Each line of the ADDONS file is the form "name;id;private" @@ -43,72 +55,57 @@ fi # case "$1" in - addons) - cut -d";" -f1 $ADDONS_FILE - ;; - addons:add) - ADDON=$3 - if [[ -z $3 ]]; then - die "You must specify an addon name" - elif [ ! -d "$ADDONS_ROOT/$ADDON" ]; then - die "Addon $ADDON does not exist" - fi - - if cut -d";" -f1 $ADDONS_FILE | grep -q "^$ADDON$"; then - die "App $APP already has addon $ADDON" - fi - - id_private=$($ADDONS_ROOT/$ADDON/provision $APP) - - echo "$ADDON;$id_private" >> $ADDONS_FILE - - ;; - addons:remove) - ADDON=$3 - if [[ -z $3 ]]; then - die "You must specify an addon name" - exit 1 - elif [ ! -d "$ADDONS_ROOT/$ADDON" ]; then - die "Addon $ADDON does not exist" - exit 1 - fi - - line=$(grep "^$ADDON;" $ADDONS_FILE) || { - die "App $APP does not have addon $ADDON" - } - - id=$(cut -d";" -f2 <<< "$line") - key=DOKKU_${ADDON^^}_URL - - $ADDONS_ROOT/$ADDON/unprovision $id - sed -i "/^$ADDON;/d" $ADDONS_FILE - sed "/^export $key=/d" $ENV_FILE | sort > $ENV_FILE - - - ;; - - addons:url) - ADDON=$3 - if [[ -z $3 ]]; then - die "You must specify an addon name" - exit 1 - elif [ ! -d "$ADDONS_ROOT/$ADDON" ]; then - die "Addon $ADDON does not exist" - exit 1 - fi - - line=$(grep "^$ADDON;" $ADDONS_FILE) || { - die "App $APP does not have addon $ADDON" - } - - id=$(cut -d";" -f2 <<< "$line") - private=$(cut -d";" -f3 <<< "$line") - - $ADDONS_ROOT/$ADDON/url $id $private - - ;; - - help) - ;; + addons) + check_app $2 + cut -d";" -f1 $ADDONS_FILE + ;; + addons:add) + check_app $2 + check_addon $3 + + if cut -d";" -f1 $ADDONS_FILE | grep -q "^$ADDON$"; then + die "App $APP already has addon $ADDON" + fi + + id_private=$($ADDONS_PATH/$ADDON/provision $APP) + + echo "$ADDON;$id_private" >> $ADDONS_FILE + + ;; + + addons:remove) + check_app $2 + check_addon $3 + + line=$(grep "^$ADDON;" $ADDONS_FILE) || { + die "App $APP does not have addon $ADDON" + } + + id=$(cut -d";" -f2 <<< "$line") + key=DOKKU_${ADDON^^}_URL + + $ADDONS_PATH/$ADDON/unprovision $id + sed -i "/^$ADDON;/d" $ADDONS_FILE + sed "/^export $key=/d" $ENV_FILE | sort > $ENV_FILE + + ;; + + addons:url) + check_app $2 + check_addon $3 + + line=$(grep "^$ADDON;" $ADDONS_FILE) || { + die "App $APP does not have addon $ADDON" + } + + id=$(cut -d";" -f2 <<< "$line") + private=$(cut -d";" -f3 <<< "$line") + + $ADDONS_PATH/$ADDON/url $id $private + + ;; + + help) + ;; esac From a18ad34eba0c8db53cef65cf1ff27af59572f3ff Mon Sep 17 00:00:00 2001 From: Paul Lietar Date: Tue, 5 Nov 2013 21:32:08 +0000 Subject: [PATCH 04/20] Add a help message to the add-on plugin. --- plugins/addons/commands | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/plugins/addons/commands b/plugins/addons/commands index 30f8faf3136..83fb18aa7a1 100755 --- a/plugins/addons/commands +++ b/plugins/addons/commands @@ -106,6 +106,13 @@ case "$1" in ;; help) + cat && cat< List add-ons enabled for an application + addons:add Add an add-on to an application + addons:remove Remove an add-on from an application (WARNING: may cause data loss) + addons:url Print the url export by an add-on +EOF + ;; esac From 2ba1286eb06e8dd4538f438a9f0f21188502546a Mon Sep 17 00:00:00 2001 From: Paul Lietar Date: Tue, 5 Nov 2013 22:02:53 +0000 Subject: [PATCH 05/20] Sort add-ons list and remove trailing whitespaces in pre-release. --- plugins/addons/commands | 2 +- plugins/addons/pre-release | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/plugins/addons/commands b/plugins/addons/commands index 83fb18aa7a1..f421a519bb0 100755 --- a/plugins/addons/commands +++ b/plugins/addons/commands @@ -57,7 +57,7 @@ function check_addon case "$1" in addons) check_app $2 - cut -d";" -f1 $ADDONS_FILE + cut -d";" -f1 $ADDONS_FILE | sort ;; addons:add) check_app $2 diff --git a/plugins/addons/pre-release b/plugins/addons/pre-release index 1e9f18cb694..06958b1163c 100755 --- a/plugins/addons/pre-release +++ b/plugins/addons/pre-release @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x -ADDONS_ROOT=/var/lib/dokku/addons +ADDONS_PATH=/var/lib/dokku/addons if [[ -z $1 ]]; then exit 1 @@ -12,7 +12,7 @@ else # Check if app exists with the same name if [ ! -d "$DOKKU_ROOT/$APP" ]; then - exit 1 + exit 1 fi touch $ADDONS_FILE @@ -21,20 +21,19 @@ fi while read line -do +do ADDON=$(cut -d";" -f1 <<< "$line") id=$(cut -d";" -f2 <<< "$line") private=$(cut -d";" -f3 <<< "$line") key=DOKKU_${ADDON^^}_URL - if [ -d "$ADDONS_ROOT/$ADDON" ]; then - url=$($ADDONS_ROOT/$ADDON/url $id $private) + if [ -d "$ADDONS_PATH/$ADDON" ]; then + url=$($ADDONS_PATH/$ADDON/url $id $private) sed "/^export $key=/d" $ENV_FILE | cat - <( echo "export $key=\"$url\"") | sort > $ENV_FILE fi - done < $ADDONS_FILE From 85fed4b9a137eaeb2494905d2ac38db44f939e93 Mon Sep 17 00:00:00 2001 From: Paul Lietar Date: Tue, 5 Nov 2013 22:04:14 +0000 Subject: [PATCH 06/20] Unprovision all add-ons in 'pre-delete'. --- plugins/addons/pre-delete | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/plugins/addons/pre-delete b/plugins/addons/pre-delete index 4ef0a84ccc2..2a2259d5478 100755 --- a/plugins/addons/pre-delete +++ b/plugins/addons/pre-delete @@ -1,3 +1,34 @@ #!/usr/bin/env bash set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x +ADDONS_PATH=/var/lib/dokku/addons + +if [[ -z $1 ]]; then + exit 1 +else + APP="$1" + ADDONS_FILE="$DOKKU_ROOT/$APP/ADDONS" + + # Check if app exists with the same name + if [ ! -d "$DOKKU_ROOT/$APP" ]; then + exit 1 + fi +fi + +if [ ! -f "$ADDONS_FILE" ]; then + exit 0 # If no addons were used just stop here +fi + +while read line +do + ADDON=$(cut -d";" -f1 <<< "$line") + id=$(cut -d";" -f2 <<< "$line") + export ADDON_ROOT=$DOKKU_ROOT/.addons/$ADDON + + mkdir -p $ADDON_ROOT + + $ADDONS_PATH/$ADDON/unprovision $id + + # We don't care about updating the ENV file, it will be delted anyway +done < $ADDONS_FILE + From 75ed755c622ee5e44e1396a281a152c50bdad32a Mon Sep 17 00:00:00 2001 From: Paul Lietar Date: Tue, 5 Nov 2013 22:50:13 +0000 Subject: [PATCH 07/20] Fix a bug where the add-ons plugin would delete all ENV file. --- plugins/addons/pre-release | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/addons/pre-release b/plugins/addons/pre-release index 06958b1163c..776fc25e97d 100755 --- a/plugins/addons/pre-release +++ b/plugins/addons/pre-release @@ -31,7 +31,7 @@ do if [ -d "$ADDONS_PATH/$ADDON" ]; then url=$($ADDONS_PATH/$ADDON/url $id $private) - sed "/^export $key=/d" $ENV_FILE | cat - <( echo "export $key=\"$url\"") | sort > $ENV_FILE + sed "/^export $key=/d" $ENV_FILE | cat - <( echo "export $key=\"$url\"") | sort -o $ENV_FILE fi done < $ADDONS_FILE From f47c91c7e4bdd18418c6a061de13a0cbf11d70df Mon Sep 17 00:00:00 2001 From: Paul Lietar Date: Tue, 5 Nov 2013 23:09:14 +0000 Subject: [PATCH 08/20] Add enable/disable to the add-on plugin. Add-ons must be enabled before they are used. This allows add-ons to install dependencies, allocate necessary resources, ... --- ADDONS.md | 25 +++++++++------ plugins/addons/commands | 70 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 82 insertions(+), 13 deletions(-) diff --git a/ADDONS.md b/ADDONS.md index 1f4c4cf8c28..0cf2b46e899 100644 --- a/ADDONS.md +++ b/ADDONS.md @@ -1,35 +1,42 @@ # User interface -Four commands are available to manage plugins : -* `dokku addons` Lists all addons an application has. -* `dokku addons:add` Adds an addon to an application. This causes the addon to allocate resources for this application (eg an account, database, ..) -* `dokku addons:url` Shows the URL a plugin has given. This is only meant for debugging. In production, there's no need to read this url manually, as it is provided in an environment variable. -* `dokku addons:remove` Removes an addon from an application. This causes the addon to destroy all associated resources. +A few commands are available to manage add-ons : +* `dokku addons` Lists all add-ons an application has. +* `dokku addons:enable` Enable a add-on. This installs required dependencies, creates required files, ... +* `dokku addons:disable` Remove a add-on from all applications and disable it. This removes unnecessary files, ... +* `dokku addons:add` Adds an add-on to an application. This causes the add-on to allocate resources for this application (eg an account, database, ..). The add-on must be enabled first. +* `dokku addons:remove` Removes an add-on from an application. This causes the add-on to destroy all associated resources. +* `dokku addons:url` Shows the URL an add-on has provided. This is only meant for debugging. In production, there's no need to read this url manually, as it is set automatically in an environment variable. The URL is passed to the application through the `DOKKU_${ADDON}_URL` where `${ADDON}` is the name of the add-on in uppercase. # Add-on development -Add-ons are located in `/var/lib/dokku/addons/`. Every subfolder is an add-on. +Add-ons are located in `/var/lib/dokku/add-ons/`. Every subfolder is an add-on. When they provision an app, add-ons should generate a unique ID, and a "private" value. The ID is used to identify the app within the add-on. This would typically be a username or database name. The "private" value is used to generate the URL. This would be a password for example. IDs and private values can take any value. They however must not include semicolons, as it is used internally by the plugin. (See the internal section) -Add-ons must provide three executable files : +Add-ons must provide a few executable files : +* `$ADDON/enable` This script takes no arguments. It should setup everything the add-on needs, eg install dependencies, create docker containers, ... +* `$ADDON/disable` This script takes no arguments. It should clean up all the data used by the add-on, eg destroy docker containers. All applications have already been unprovisonised before this script is called. * `$ADDON/provision` This script takes one argument, the app name. It should only be used to generate understandable IDs, which contain the app name. Add-ons shouldn't use it the access the app's config files, ... The script should output the generated id and private value on stdout, separated by a semicolon. * `$ADDON/unprovision` This script takes one argument, the ID. It should destroy all resources associated with this ID. * `$ADDON/url` This script takes two arguments, the ID and the private value. It should output the url on stdout. +Apart from `$ADDON/enable` all scripts are run only with the add-on in enabled state. Add-on's are free to run the service in the way they like. They can run it on the cloud, on the host, on a single docker container or on a container per provisioned app. The plugin doesn't care, as long as the add-on provides a URL which is accessible. Because URLs might change (docker can assign different IPs/ports after reboot), the `url` script is called each time the app is released. (BTW, this means we should release all apps at startup, rather than deploy them) ## Guidelines ### Add-ons internal files Add-ons may want to store files for their own internal use, for eg. database storage. -Before any add-on script is ran, a `$ADDON_ROOT` environment variable is exported, which contains the path to a directory where the addon can safely save any file. The directory is guaranteed to exist before add-on scripts are ran. -Currently it is set to `$DOKKU_ROOT/.addons/$ADDON` where `$ADDON` is the addon's name. However, this might change, therefore plugins should use the `$ADDON_ROOT` variable rather than hardcoding it. +Before any add-on script is ran, a `$ADDON_ROOT` environment variable is exported, which contains the path to a directory where the add-on can safely save any file. The directory is guaranteed to exist before add-on scripts are ran. +Currently it is set to `$DOKKU_ROOT/.add-ons/$ADDON` where `$ADDON` is the add-on's name. However, this might change, therefore add-ons should use the `$ADDON_ROOT` variable rather than hardcoding it. +The add-on plugin automatically creates and removes a `$ADDON_ROOT/enabled` file to keep track of which plugin is installed. Add-ons mustn't create, modify nor delete this file themselves. +It is created after the `$ADDON/enable` script completes, and deleted after the `$ADDON/disabled` script completes. # Internals The add-on plugin uses the `$APP/ADDONS` file to save which add-ons are in use. diff --git a/plugins/addons/commands b/plugins/addons/commands index f421a519bb0..debb3510440 100755 --- a/plugins/addons/commands +++ b/plugins/addons/commands @@ -47,7 +47,6 @@ function check_addon mkdir -p $ADDON_ROOT } - # # Each line of the ADDONS file is the form "name;id;private" # @@ -56,13 +55,29 @@ function check_addon case "$1" in addons) - check_app $2 - cut -d";" -f1 $ADDONS_FILE | sort + if [[ -z $2 || $2 == "-a" ]]; then + for ADDON in $(ls -d $ADDONS_PATH/*/ | tr '\n' '\0' | xargs -0 -n 1 basename); do + ADDON_ROOT=$DOKKU_ROOT/.addons/$ADDON + + if [ -f $ADDON_ROOT/enabled ]; then + echo "$ADDON" + elif [[ $2 == "-a" ]]; then + echo "$ADDON (disabled)" + fi + done + else + check_app $2 + cut -d";" -f1 $ADDONS_FILE | sort + fi ;; addons:add) check_app $2 check_addon $3 + if [ ! -f $ADDON_ROOT/enabled ]; then + die "Add-on $ADDON is not enabled" + fi + if cut -d";" -f1 $ADDONS_FILE | grep -q "^$ADDON$"; then die "App $APP already has addon $ADDON" fi @@ -90,6 +105,50 @@ case "$1" in ;; + addons:enable) + check_addon $2 + if [ -f $ADDON_ROOT/enabled ]; then + die "Add-on $ADDON is already enabled" + fi + + $ADDONS_PATH/$ADDON/enable + + touch $ADDON_ROOT/enabled + + ;; + + addons:disable) + check_addon $2 + if [ ! -f $ADDON_ROOT/enabled ]; then + die "Add-on $ADDON is not enabled" + fi + + # Unprovision all apps + for APP in $(ls -d $DOKKU_ROOT/*/ | tr '\n' '\0' | xargs -0 -n 1 basename); do + ADDONS_FILE="$DOKKU_ROOT/$APP/ADDONS" + ENV_FILE="$DOKKU_ROOT/$APP/ENV" + + touch $ADDONS_FILE + touch $ENV_FILE + + line=$(grep "^$ADDON;" $ADDONS_FILE) || { + continue + } + + id=$(cut -d";" -f2 <<< "$line") + key=DOKKU_${ADDON^^}_URL + + $ADDONS_PATH/$ADDON/unprovision $id + sed -i "/^$ADDON;/d" $ADDONS_FILE + sed "/^export $key=/d" $ENV_FILE | sort > $ENV_FILE + done + + $ADDONS_PATH/$ADDON/disable + + rm $ADDON_ROOT/enabled + + ;; + addons:url) check_app $2 check_addon $3 @@ -107,7 +166,10 @@ case "$1" in help) cat && cat< List add-ons enabled for an application + addons [-a] List enabled add-ons. If -a is specified, disabled add-ons are listed as well. + addons List all add-ons used by an application + addons:enable Enable an add-on + addons:disable Disable an add-on (WARNING: may cause data loss) addons:add Add an add-on to an application addons:remove Remove an add-on from an application (WARNING: may cause data loss) addons:url Print the url export by an add-on From 058a556716f66585ac33941fdf772c3c464a37e0 Mon Sep 17 00:00:00 2001 From: Paul Lietar Date: Wed, 6 Nov 2013 04:46:14 +0000 Subject: [PATCH 09/20] export ADDON_ROOT in the pre-release hook --- plugins/addons/pre-release | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/addons/pre-release b/plugins/addons/pre-release index 776fc25e97d..4125a45b43f 100755 --- a/plugins/addons/pre-release +++ b/plugins/addons/pre-release @@ -26,6 +26,9 @@ do id=$(cut -d";" -f2 <<< "$line") private=$(cut -d";" -f3 <<< "$line") + export ADDON_ROOT=$DOKKU_ROOT/.addons/$ADDON + mkdir -p $ADDON_ROOT + key=DOKKU_${ADDON^^}_URL if [ -d "$ADDONS_PATH/$ADDON" ]; then From 60f57567af1951f9dd23792b5d6d655721357d06 Mon Sep 17 00:00:00 2001 From: Paul Lietar Date: Thu, 7 Nov 2013 00:55:50 +0000 Subject: [PATCH 10/20] Refactorize a lot of code, and rename ADDON_ROOT to ADDON_DATA. --- ADDONS.md | 7 ++- plugins/addons/addons-common.sh | 101 ++++++++++++++++++++++++++++++++ plugins/addons/commands | 98 ++++++------------------------- plugins/addons/pre-delete | 27 +++------ plugins/addons/pre-release | 31 +++------- 5 files changed, 137 insertions(+), 127 deletions(-) create mode 100644 plugins/addons/addons-common.sh diff --git a/ADDONS.md b/ADDONS.md index 0cf2b46e899..2e737363cbe 100644 --- a/ADDONS.md +++ b/ADDONS.md @@ -32,10 +32,10 @@ Because URLs might change (docker can assign different IPs/ports after reboot), ## Guidelines ### Add-ons internal files Add-ons may want to store files for their own internal use, for eg. database storage. -Before any add-on script is ran, a `$ADDON_ROOT` environment variable is exported, which contains the path to a directory where the add-on can safely save any file. The directory is guaranteed to exist before add-on scripts are ran. -Currently it is set to `$DOKKU_ROOT/.add-ons/$ADDON` where `$ADDON` is the add-on's name. However, this might change, therefore add-ons should use the `$ADDON_ROOT` variable rather than hardcoding it. +Before any add-on script is ran, a `$ADDON_DATA` environment variable is exported, which contains the path to a directory where the add-on can safely save any file. The directory is guaranteed to exist before add-on scripts are ran. +Currently it is set to `$DOKKU_ROOT/.add-ons/$ADDON` where `$ADDON` is the add-on's name. However, this might change, therefore add-ons should use the `$ADDON_DATA` variable rather than hardcoding it. -The add-on plugin automatically creates and removes a `$ADDON_ROOT/enabled` file to keep track of which plugin is installed. Add-ons mustn't create, modify nor delete this file themselves. +The add-on plugin automatically creates and removes a `$ADDON_DATA/enabled` file to keep track of which plugin is installed. Add-ons mustn't create, modify nor delete this file themselves. It is created after the `$ADDON/enable` script completes, and deleted after the `$ADDON/disabled` script completes. # Internals @@ -45,3 +45,4 @@ Each line has the following format : name;id;private In the `pre-release` hook, the plugin generates environment variables based on the ids and private values, using the `url` script, and saves them to `$APP/ENV` + diff --git a/plugins/addons/addons-common.sh b/plugins/addons/addons-common.sh new file mode 100644 index 00000000000..ea6adfc34fe --- /dev/null +++ b/plugins/addons/addons-common.sh @@ -0,0 +1,101 @@ + +ADDONS_PATH=/var/lib/dokku/addons + +function die +{ + $QUIET || echo $* >&2 + exit 1 +} + +function export_addon_vars() +{ + export ADDON="$1" + export ADDON_DATA="$DOKKU_ROOT/.addons/$ADDON" + export ADDON_ROOT="$ADDONS_PATH/$ADDON" +} + +function check_app +{ + if [[ -z $1 ]]; then + die "You must specify an app name" + else + APP="$1" + ADDONS_FILE="$DOKKU_ROOT/$APP/ADDONS" + ENV_FILE="$DOKKU_ROOT/$APP/ENV" + + # Check if app exists with the same name + if [ ! -d "$DOKKU_ROOT/$APP" ]; then + die "App $APP does not exist" + fi + + [ -f $ADDONS_FILE ] || { + $QUIET || echo "-----> Creating $ADDONS_FILE" + touch $ADDONS_FILE + } + [ -f $ENV_FILE ] || { + $QUIET || echo "-----> Creating $ENV_FILE" + touch $ENV_FILE + } + fi +} + +function check_addon +{ + if [[ -z $1 ]]; then + die "You must specify an addon name" + elif [ ! -d "$ADDONS_PATH/$1" ]; then + die "Addon $1 does not exist" + fi + export_addon_vars $1 + + mkdir -p $ADDON_DATA +} + +function check_addon_enabled +{ + check_addon $1 + if [ ! -f $ADDON_DATA/enabled ]; then + die "Add-on $ADDON is not enabled" + fi +} + +function check_addon_disabled +{ + check_addon $1 + if [ -f $ADDON_DATA/enabled ]; then + die "Add-on $ADDON is already enabled" + fi +} + +function check_addon_provisioned +{ + local line + line=$(grep "^$ADDON;" $ADDONS_FILE) || { + die "App $APP does not have addon $ADDON" + } + + split_addon_line $line _ ADDON_ID ADDON_PRIVATE +} + +function check_addon_unprovisioned +{ + if grep -q "^$ADDON;" $ADDONS_FILE; then + die "App $APP already has addon $ADDON" + fi +} + +function split_addon_line +{ + parts=($(echo $1 | sed 's/;/ /g')) + if [[ ! -z $2 && $2 != "_" ]]; then + eval "$2=${parts[0]}" + fi + if [[ ! -z $3 && $3 != "_" ]]; then + eval "$3=${parts[1]}" + fi + if [[ ! -z $4 && $4 != "_" ]]; then + eval "$4=${parts[2]}" + fi +} + + diff --git a/plugins/addons/commands b/plugins/addons/commands index debb3510440..a5ef5a00f45 100755 --- a/plugins/addons/commands +++ b/plugins/addons/commands @@ -1,51 +1,10 @@ #!/usr/bin/env bash set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x -ADDONS_PATH=/var/lib/dokku/addons - -function die -{ - echo $* >&2 - exit 1 -} - -function check_app -{ - if [[ -z $1 ]]; then - die "You must specify an app name" - else - APP="$1" - ADDONS_FILE="$DOKKU_ROOT/$APP/ADDONS" - ENV_FILE="$DOKKU_ROOT/$APP/ENV" - - # Check if app exists with the same name - if [ ! -d "$DOKKU_ROOT/$APP" ]; then - die "App $APP does not exist" - fi +QUIET=false + +source $(dirname ${BASH_SOURCE[0]})/addons-common.sh - [ -f $ADDONS_FILE ] || { - echo "-----> Creating $ADDONS_FILE" - touch $ADDONS_FILE - } - [ -f $ENV_FILE ] || { - echo "-----> Creating $ENV_FILE" - touch $ENV_FILE - } - fi -} - -function check_addon -{ - if [[ -z $1 ]]; then - die "You must specify an addon name" - elif [ ! -d "$ADDONS_PATH/$ADDON" ]; then - die "Addon $ADDON does not exist" - fi - ADDON=$1 - export ADDON_ROOT=$DOKKU_ROOT/.addons/$ADDON - - mkdir -p $ADDON_ROOT -} # # Each line of the ADDONS file is the form "name;id;private" @@ -57,9 +16,9 @@ case "$1" in addons) if [[ -z $2 || $2 == "-a" ]]; then for ADDON in $(ls -d $ADDONS_PATH/*/ | tr '\n' '\0' | xargs -0 -n 1 basename); do - ADDON_ROOT=$DOKKU_ROOT/.addons/$ADDON + ADDON_DATA=$DOKKU_ROOT/.addons/$ADDON - if [ -f $ADDON_ROOT/enabled ]; then + if [ -f $ADDON_DATA/enabled ]; then echo "$ADDON" elif [[ $2 == "-a" ]]; then echo "$ADDON (disabled)" @@ -72,15 +31,8 @@ case "$1" in ;; addons:add) check_app $2 - check_addon $3 - - if [ ! -f $ADDON_ROOT/enabled ]; then - die "Add-on $ADDON is not enabled" - fi - - if cut -d";" -f1 $ADDONS_FILE | grep -q "^$ADDON$"; then - die "App $APP already has addon $ADDON" - fi + check_addon_enabled $3 + check_addon_unprovisioned id_private=$($ADDONS_PATH/$ADDON/provision $APP) @@ -91,37 +43,27 @@ case "$1" in addons:remove) check_app $2 check_addon $3 + check_addon_provisioned - line=$(grep "^$ADDON;" $ADDONS_FILE) || { - die "App $APP does not have addon $ADDON" - } - - id=$(cut -d";" -f2 <<< "$line") key=DOKKU_${ADDON^^}_URL - $ADDONS_PATH/$ADDON/unprovision $id + $ADDON_ROOT/unprovision $ADDON_ID sed -i "/^$ADDON;/d" $ADDONS_FILE sed "/^export $key=/d" $ENV_FILE | sort > $ENV_FILE ;; addons:enable) - check_addon $2 - if [ -f $ADDON_ROOT/enabled ]; then - die "Add-on $ADDON is already enabled" - fi + check_addon_disabled $2 - $ADDONS_PATH/$ADDON/enable + $ADDON_ROOT/enable - touch $ADDON_ROOT/enabled + touch $ADDON_DATA/enabled ;; addons:disable) - check_addon $2 - if [ ! -f $ADDON_ROOT/enabled ]; then - die "Add-on $ADDON is not enabled" - fi + check_addon_enabled $2 # Unprovision all apps for APP in $(ls -d $DOKKU_ROOT/*/ | tr '\n' '\0' | xargs -0 -n 1 basename); do @@ -138,29 +80,23 @@ case "$1" in id=$(cut -d";" -f2 <<< "$line") key=DOKKU_${ADDON^^}_URL - $ADDONS_PATH/$ADDON/unprovision $id + $ADDON_ROOT/unprovision $id sed -i "/^$ADDON;/d" $ADDONS_FILE sed "/^export $key=/d" $ENV_FILE | sort > $ENV_FILE done $ADDONS_PATH/$ADDON/disable - rm $ADDON_ROOT/enabled + rm $ADDON_DATA/enabled ;; addons:url) check_app $2 check_addon $3 + check_addon_provisioned - line=$(grep "^$ADDON;" $ADDONS_FILE) || { - die "App $APP does not have addon $ADDON" - } - - id=$(cut -d";" -f2 <<< "$line") - private=$(cut -d";" -f3 <<< "$line") - - $ADDONS_PATH/$ADDON/url $id $private + $ADDON_ROOT/url $ADDON_ID $ADDON_PRIVATE ;; diff --git a/plugins/addons/pre-delete b/plugins/addons/pre-delete index 2a2259d5478..20a41a8fffa 100755 --- a/plugins/addons/pre-delete +++ b/plugins/addons/pre-delete @@ -1,33 +1,20 @@ #!/usr/bin/env bash set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x -ADDONS_PATH=/var/lib/dokku/addons +QUIET=true -if [[ -z $1 ]]; then - exit 1 -else - APP="$1" - ADDONS_FILE="$DOKKU_ROOT/$APP/ADDONS" +source $(dirname ${BASH_SOURCE[0]})/addons-common.sh - # Check if app exists with the same name - if [ ! -d "$DOKKU_ROOT/$APP" ]; then - exit 1 - fi -fi - -if [ ! -f "$ADDONS_FILE" ]; then - exit 0 # If no addons were used just stop here -fi +check_app $1 while read line do - ADDON=$(cut -d";" -f1 <<< "$line") - id=$(cut -d";" -f2 <<< "$line") - export ADDON_ROOT=$DOKKU_ROOT/.addons/$ADDON + split_addon_line $line ADDON id + export_addon_vars $ADDON - mkdir -p $ADDON_ROOT + mkdir -p $ADDON_DATA - $ADDONS_PATH/$ADDON/unprovision $id + $ADDON_ROOT/unprovision $id # We don't care about updating the ENV file, it will be delted anyway done < $ADDONS_FILE diff --git a/plugins/addons/pre-release b/plugins/addons/pre-release index 4125a45b43f..140fc66fc4c 100755 --- a/plugins/addons/pre-release +++ b/plugins/addons/pre-release @@ -1,38 +1,23 @@ #!/usr/bin/env bash set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x -ADDONS_PATH=/var/lib/dokku/addons - -if [[ -z $1 ]]; then - exit 1 -else - APP="$1" - ENV_FILE="$DOKKU_ROOT/$APP/ENV" - ADDONS_FILE="$DOKKU_ROOT/$APP/ADDONS" - - # Check if app exists with the same name - if [ ! -d "$DOKKU_ROOT/$APP" ]; then - exit 1 - fi +QUIET=true - touch $ADDONS_FILE - touch $ENV_FILE -fi +source $(dirname ${BASH_SOURCE[0]})/addons-common.sh +check_app $1 while read line do - ADDON=$(cut -d";" -f1 <<< "$line") - id=$(cut -d";" -f2 <<< "$line") - private=$(cut -d";" -f3 <<< "$line") + split_addon_line $line ADDON id private + export_addon_vars $ADDON - export ADDON_ROOT=$DOKKU_ROOT/.addons/$ADDON - mkdir -p $ADDON_ROOT + mkdir -p $ADDON_DATA key=DOKKU_${ADDON^^}_URL - if [ -d "$ADDONS_PATH/$ADDON" ]; then - url=$($ADDONS_PATH/$ADDON/url $id $private) + if [ -d "$ADDON_ROOT" ]; then + url=$($ADDON_ROOT/url $id $private) sed "/^export $key=/d" $ENV_FILE | cat - <( echo "export $key=\"$url\"") | sort -o $ENV_FILE fi From e6b58b7efaccb68b6018bbcefe1116450038e9ff Mon Sep 17 00:00:00 2001 From: Paul Lietar Date: Fri, 15 Nov 2013 10:03:17 +0000 Subject: [PATCH 11/20] Replace the semicolon by a colon in the ADDONS file. --- plugins/addons/addons-common.sh | 7 +++---- plugins/addons/commands | 14 +++++++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/plugins/addons/addons-common.sh b/plugins/addons/addons-common.sh index ea6adfc34fe..e7168f1f9ef 100644 --- a/plugins/addons/addons-common.sh +++ b/plugins/addons/addons-common.sh @@ -70,7 +70,7 @@ function check_addon_disabled function check_addon_provisioned { local line - line=$(grep "^$ADDON;" $ADDONS_FILE) || { + line=$(grep "^$ADDON:" $ADDONS_FILE) || { die "App $APP does not have addon $ADDON" } @@ -79,14 +79,14 @@ function check_addon_provisioned function check_addon_unprovisioned { - if grep -q "^$ADDON;" $ADDONS_FILE; then + if grep -q "^$ADDON:" $ADDONS_FILE; then die "App $APP already has addon $ADDON" fi } function split_addon_line { - parts=($(echo $1 | sed 's/;/ /g')) + parts=($(echo $1 | sed 's/:/ /g')) if [[ ! -z $2 && $2 != "_" ]]; then eval "$2=${parts[0]}" fi @@ -98,4 +98,3 @@ function split_addon_line fi } - diff --git a/plugins/addons/commands b/plugins/addons/commands index a5ef5a00f45..10017b4a393 100755 --- a/plugins/addons/commands +++ b/plugins/addons/commands @@ -7,7 +7,7 @@ source $(dirname ${BASH_SOURCE[0]})/addons-common.sh # -# Each line of the ADDONS file is the form "name;id;private" +# Each line of the ADDONS file is the form "name:id:private" # # The combination of the id and the private data is used by the addon to generate the url # @@ -26,7 +26,7 @@ case "$1" in done else check_app $2 - cut -d";" -f1 $ADDONS_FILE | sort + cut -d":" -f1 $ADDONS_FILE | sort fi ;; addons:add) @@ -36,7 +36,7 @@ case "$1" in id_private=$($ADDONS_PATH/$ADDON/provision $APP) - echo "$ADDON;$id_private" >> $ADDONS_FILE + echo "$ADDON:$id_private" >> $ADDONS_FILE ;; @@ -48,7 +48,7 @@ case "$1" in key=DOKKU_${ADDON^^}_URL $ADDON_ROOT/unprovision $ADDON_ID - sed -i "/^$ADDON;/d" $ADDONS_FILE + sed -i "/^$ADDON:/d" $ADDONS_FILE sed "/^export $key=/d" $ENV_FILE | sort > $ENV_FILE ;; @@ -73,15 +73,15 @@ case "$1" in touch $ADDONS_FILE touch $ENV_FILE - line=$(grep "^$ADDON;" $ADDONS_FILE) || { + line=$(grep "^$ADDON:" $ADDONS_FILE) || { continue } - id=$(cut -d";" -f2 <<< "$line") + id=$(cut -d":" -f2 <<< "$line") key=DOKKU_${ADDON^^}_URL $ADDON_ROOT/unprovision $id - sed -i "/^$ADDON;/d" $ADDONS_FILE + sed -i "/^$ADDON:/d" $ADDONS_FILE sed "/^export $key=/d" $ENV_FILE | sort > $ENV_FILE done From 95f95b61e739324086b71693aa23cd655de8de24 Mon Sep 17 00:00:00 2001 From: Paul Lietar Date: Mon, 25 Nov 2013 23:17:11 +0000 Subject: [PATCH 12/20] Add a few comments in addons-common. --- plugins/addons/addons-common.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/plugins/addons/addons-common.sh b/plugins/addons/addons-common.sh index e7168f1f9ef..e4a1d01ff3b 100644 --- a/plugins/addons/addons-common.sh +++ b/plugins/addons/addons-common.sh @@ -14,6 +14,8 @@ function export_addon_vars() export ADDON_ROOT="$ADDONS_PATH/$ADDON" } +# Check if an app exists. +# If it does export APP, ADDONS_FILE and ENV_FILE function check_app { if [[ -z $1 ]]; then @@ -39,6 +41,8 @@ function check_app fi } +# Check if an addon exists. +# if it does, export ADDON, ADDON_DATA and ADDON_ROOT function check_addon { if [[ -z $1 ]]; then @@ -51,6 +55,8 @@ function check_addon mkdir -p $ADDON_DATA } +# Check if an addon exists and is enabled. +# See check_addon for exported variables function check_addon_enabled { check_addon $1 @@ -59,6 +65,8 @@ function check_addon_enabled fi } +# Check if an addon exists and is disabled. +# See check_addon for exported variables function check_addon_disabled { check_addon $1 @@ -67,6 +75,9 @@ function check_addon_disabled fi } +# Make sure the addon is provisioned. +# check_app and check_addon must be called first. +# Exports ADDON_ID and ADDON_PRIVATE function check_addon_provisioned { local line @@ -77,6 +88,8 @@ function check_addon_provisioned split_addon_line $line _ ADDON_ID ADDON_PRIVATE } +# Make sure the addon is not provisioned. +# check_app and check_addon must be called first. function check_addon_unprovisioned { if grep -q "^$ADDON:" $ADDONS_FILE; then From b7b3fad2484a8f8a47720c205b67c72f28945ba5 Mon Sep 17 00:00:00 2001 From: Paul Lietar Date: Mon, 25 Nov 2013 23:57:37 +0000 Subject: [PATCH 13/20] Move and split the ADDONS.md file. --- ADDONS.md | 48 ---------------------------------------- docs/using-addons.md | 50 ++++++++++++++++++++++++++++++++++++++++++ docs/writing-addons.md | 30 +++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 48 deletions(-) delete mode 100644 ADDONS.md create mode 100644 docs/using-addons.md create mode 100644 docs/writing-addons.md diff --git a/ADDONS.md b/ADDONS.md deleted file mode 100644 index 2e737363cbe..00000000000 --- a/ADDONS.md +++ /dev/null @@ -1,48 +0,0 @@ -# User interface -A few commands are available to manage add-ons : -* `dokku addons` Lists all add-ons an application has. -* `dokku addons:enable` Enable a add-on. This installs required dependencies, creates required files, ... -* `dokku addons:disable` Remove a add-on from all applications and disable it. This removes unnecessary files, ... -* `dokku addons:add` Adds an add-on to an application. This causes the add-on to allocate resources for this application (eg an account, database, ..). The add-on must be enabled first. -* `dokku addons:remove` Removes an add-on from an application. This causes the add-on to destroy all associated resources. -* `dokku addons:url` Shows the URL an add-on has provided. This is only meant for debugging. In production, there's no need to read this url manually, as it is set automatically in an environment variable. - -The URL is passed to the application through the `DOKKU_${ADDON}_URL` where `${ADDON}` is the name of the add-on in uppercase. - -# Add-on development -Add-ons are located in `/var/lib/dokku/add-ons/`. Every subfolder is an add-on. -When they provision an app, add-ons should generate a unique ID, and a "private" value. -The ID is used to identify the app within the add-on. This would typically be a username or database name. -The "private" value is used to generate the URL. This would be a password for example. - -IDs and private values can take any value. They however must not include semicolons, as it is used internally by the plugin. (See the internal section) - -Add-ons must provide a few executable files : -* `$ADDON/enable` This script takes no arguments. It should setup everything the add-on needs, eg install dependencies, create docker containers, ... -* `$ADDON/disable` This script takes no arguments. It should clean up all the data used by the add-on, eg destroy docker containers. All applications have already been unprovisonised before this script is called. -* `$ADDON/provision` This script takes one argument, the app name. It should only be used to generate understandable IDs, which contain the app name. Add-ons shouldn't use it the access the app's config files, ... -The script should output the generated id and private value on stdout, separated by a semicolon. -* `$ADDON/unprovision` This script takes one argument, the ID. It should destroy all resources associated with this ID. -* `$ADDON/url` This script takes two arguments, the ID and the private value. It should output the url on stdout. - -Apart from `$ADDON/enable` all scripts are run only with the add-on in enabled state. -Add-on's are free to run the service in the way they like. They can run it on the cloud, on the host, on a single docker container or on a container per provisioned app. The plugin doesn't care, as long as the add-on provides a URL which is accessible. -Because URLs might change (docker can assign different IPs/ports after reboot), the `url` script is called each time the app is released. (BTW, this means we should release all apps at startup, rather than deploy them) - -## Guidelines -### Add-ons internal files -Add-ons may want to store files for their own internal use, for eg. database storage. -Before any add-on script is ran, a `$ADDON_DATA` environment variable is exported, which contains the path to a directory where the add-on can safely save any file. The directory is guaranteed to exist before add-on scripts are ran. -Currently it is set to `$DOKKU_ROOT/.add-ons/$ADDON` where `$ADDON` is the add-on's name. However, this might change, therefore add-ons should use the `$ADDON_DATA` variable rather than hardcoding it. - -The add-on plugin automatically creates and removes a `$ADDON_DATA/enabled` file to keep track of which plugin is installed. Add-ons mustn't create, modify nor delete this file themselves. -It is created after the `$ADDON/enable` script completes, and deleted after the `$ADDON/disabled` script completes. - -# Internals -The add-on plugin uses the `$APP/ADDONS` file to save which add-ons are in use. -Each line has the following format : - - name;id;private - -In the `pre-release` hook, the plugin generates environment variables based on the ids and private values, using the `url` script, and saves them to `$APP/ENV` - diff --git a/docs/using-addons.md b/docs/using-addons.md new file mode 100644 index 00000000000..6db255bf842 --- /dev/null +++ b/docs/using-addons.md @@ -0,0 +1,50 @@ +# Addons +Addons provide services. They are similar to heroku's addons. + +Heroku provides a good description of how to use and manage addons. +Most commands are similar in dokku. + +https://devcenter.heroku.com/articles/managing-add-ons + +## Usage +In the following examples, `myapp` is the name of an application, `mariadb` is the addon's name. + +First, you should list all available addons : + + dokku addons -a + +An addon marked as disabled may not be used by applications. You must enable it first : + + dokku addons:enable mariadb + +Depending on the addon, you might need to type your password into sudo. + +Once the addon is enabled, add it to your application : + + dokku addons:add myapp mariadb + +Congratulations, you can now use a mariadb database in your application. +The url to which your application should connect is located in the `DOKKU_MARIADB_URL`. +You can visualize this url by running the `addons:url` command. + + dokku addons:url myapp mariadb + +The contents and format of the url is addon-specific. +Note that the indicated host is from the application's container point of view. +It may not be accessible from outside. +In the case of the mariadb addon, it has the following format : + + mysql://USER@PASSWORD/HOST:PORT/DATABASE + + +You can remove an addon from your application : + + dokku addons:remove myapp mariadb + +WARNING: This removes all data (such as a database) associated with this application. +Use with care. + +If none of your applications uses an addon, you can disable it : + + dokku addons:disable mariadb + diff --git a/docs/writing-addons.md b/docs/writing-addons.md new file mode 100644 index 00000000000..ab54e966ff8 --- /dev/null +++ b/docs/writing-addons.md @@ -0,0 +1,30 @@ +# Add-on development +Add-ons are located in `/var/lib/dokku/add-ons/`. Every subfolder is an add-on. +When they provision an app, add-ons should generate a unique ID, and a "private" value. +The ID is used to identify the app within the add-on. This would typically be a username or database name. +The "private" value is used to generate the URL. This would be a password for example. + +IDs and private values can take any value. They however must not include colons, as it is used internally by the plugin. + +Add-ons must provide a few executable files : +* `enable` This script takes no arguments. It should setup everything the add-on needs, eg install dependencies, create docker containers, ... +* `disable` This script takes no arguments. It should clean up all the data used by the add-on, eg destroy docker containers. All applications have already been unprovisonised before this script is called. +* `provision` This script takes one argument, the app name. It should only be used to generate understandable IDs, which contain the app name. Add-ons shouldn't use it the access the app's config files, ... +The script should output the generated id and private value on stdout, separated by a colon. +* `unprovision` This script takes one argument, the ID. It should destroy all resources associated with this ID. +* `url` This script takes two arguments, the ID and the private value. It should output the url on stdout. + +Apart from `$ADDON/enable` all scripts are run only with the add-on in enabled state. +Add-on's are free to run the service in the way they like. They can run it on the cloud, on the host, on a single docker container or on a container per provisioned app. The plugin doesn't care, as long as the add-on provides a URL which is accessible. +Because URLs might change (docker can assign different IPs/ports after reboot), the `url` script is called each time the app is released. + +## Guidelines +### Add-ons internal files +Add-ons may want to store files for their own internal use, for eg. database storage. +Before any add-on script is ran, a `$ADDON_DATA` environment variable is exported, which contains the path to a directory where the add-on can safely save any file. The directory is guaranteed to exist before add-on scripts are ran. +Currently it is set to `$DOKKU_ROOT/.add-ons/$ADDON` where `$ADDON` is the add-on's name. However, this might change, therefore add-ons should use the `$ADDON_DATA` variable rather than hardcoding it. + +The add-on plugin automatically creates and removes a `$ADDON_DATA/enabled` file to keep track of which plugin is installed. Add-ons mustn't create, modify nor delete this file themselves. +It is created after the `$ADDON/enable` script completes, and deleted after the `$ADDON/disabled` script completes. + +Moreover, addons may contain extra immutable data files. The `$ADDON_ROOT` variable contains the addon's installation path. This would typically be `/var/lib/dokku/addons/$ADDONS` From d1cd2195fb652b3b3a310f2a75999a54f12780fd Mon Sep 17 00:00:00 2001 From: Paul Lietar Date: Tue, 26 Nov 2013 03:55:17 +0000 Subject: [PATCH 14/20] Add the addons directory to gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a3c121f5b0e..6911ed15d25 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .vagrant .DS_Store stack.tgz +/addons From 20c55c86d0dac84103ecb64d8646e14589f7a9cc Mon Sep 17 00:00:00 2001 From: Paul Lietar Date: Wed, 27 Nov 2013 21:38:42 +0000 Subject: [PATCH 15/20] Fail when disabling an add-on which is still in use. This is safer and simpler than unprovisioning all apps. --- plugins/addons/commands | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/plugins/addons/commands b/plugins/addons/commands index 10017b4a393..cf4f1630079 100755 --- a/plugins/addons/commands +++ b/plugins/addons/commands @@ -65,25 +65,12 @@ case "$1" in addons:disable) check_addon_enabled $2 - # Unprovision all apps - for APP in $(ls -d $DOKKU_ROOT/*/ | tr '\n' '\0' | xargs -0 -n 1 basename); do - ADDONS_FILE="$DOKKU_ROOT/$APP/ADDONS" - ENV_FILE="$DOKKU_ROOT/$APP/ENV" - - touch $ADDONS_FILE - touch $ENV_FILE - - line=$(grep "^$ADDON:" $ADDONS_FILE) || { - continue - } - - id=$(cut -d":" -f2 <<< "$line") - key=DOKKU_${ADDON^^}_URL - - $ADDON_ROOT/unprovision $id - sed -i "/^$ADDON:/d" $ADDONS_FILE - sed "/^export $key=/d" $ENV_FILE | sort > $ENV_FILE - done + # Check for apps using the addon + if grep -q "^$ADDON:" $DOKKU_ROOT/*/ADDONS; then + echo "$ADDON is still used by the following applications :" 2>&1 + grep -l "^$ADDON:" $DOKKU_ROOT/*/ADDONS | sed 's#^.*/\(.*\)/ADDONS$# * \1#' 2>&1 + exit 1 + fi $ADDONS_PATH/$ADDON/disable @@ -105,7 +92,7 @@ case "$1" in addons [-a] List enabled add-ons. If -a is specified, disabled add-ons are listed as well. addons List all add-ons used by an application addons:enable Enable an add-on - addons:disable Disable an add-on (WARNING: may cause data loss) + addons:disable Disable an add-on addons:add Add an add-on to an application addons:remove Remove an add-on from an application (WARNING: may cause data loss) addons:url Print the url export by an add-on From 418a5957d520ccc3198dd9866ff7090220db064e Mon Sep 17 00:00:00 2001 From: Paul Lietar Date: Wed, 27 Nov 2013 23:28:56 +0000 Subject: [PATCH 16/20] Add a bit of output. --- plugins/addons/commands | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/plugins/addons/commands b/plugins/addons/commands index cf4f1630079..c4ab8193c21 100755 --- a/plugins/addons/commands +++ b/plugins/addons/commands @@ -5,7 +5,6 @@ QUIET=false source $(dirname ${BASH_SOURCE[0]})/addons-common.sh - # # Each line of the ADDONS file is the form "name:id:private" # @@ -15,25 +14,44 @@ source $(dirname ${BASH_SOURCE[0]})/addons-common.sh case "$1" in addons) if [[ -z $2 || $2 == "-a" ]]; then + if [[ ! -d $ADDONS_PATH || -z $(ls -A $ADDONS_PATH) ]]; then + die "No addons installed." + elif [[ -z $2 && -z $(ls $DOKKU_ROOT/.addons/*/enabled 2>/dev/null) ]]; then + die "No addons enabled." + fi + + if [[ $2 == "-a" ]]; then + echo "Installed addons :" + else + echo "Enabled addons :" + fi + for ADDON in $(ls -d $ADDONS_PATH/*/ | tr '\n' '\0' | xargs -0 -n 1 basename); do ADDON_DATA=$DOKKU_ROOT/.addons/$ADDON - if [ -f $ADDON_DATA/enabled ]; then - echo "$ADDON" + if [[ -f $ADDON_DATA/enabled ]]; then + echo " * $ADDON" elif [[ $2 == "-a" ]]; then - echo "$ADDON (disabled)" + echo " * $ADDON (disabled)" fi done else check_app $2 - cut -d":" -f1 $ADDONS_FILE | sort + if [[ -z $(< $ADDONS_FILE) ]]; then + die "Application $APP does not have any addons." + else + echo "Enabled addons for application $APP :" + cut -d":" -f1 $ADDONS_FILE | sort | sed 's/^/ * /' + fi fi ;; + addons:add) check_app $2 check_addon_enabled $3 check_addon_unprovisioned + echo "Provisioning addon $ADDON for application $APP ..." id_private=$($ADDONS_PATH/$ADDON/provision $APP) echo "$ADDON:$id_private" >> $ADDONS_FILE @@ -47,6 +65,7 @@ case "$1" in key=DOKKU_${ADDON^^}_URL + echo "Unprovisioning addon $ADDON from application $APP ..." $ADDON_ROOT/unprovision $ADDON_ID sed -i "/^$ADDON:/d" $ADDONS_FILE sed "/^export $key=/d" $ENV_FILE | sort > $ENV_FILE @@ -56,6 +75,8 @@ case "$1" in addons:enable) check_addon_disabled $2 + echo "Enabling addon $ADDON ..." + $ADDON_ROOT/enable touch $ADDON_DATA/enabled @@ -72,6 +93,8 @@ case "$1" in exit 1 fi + echo "Disabling addon $ADDON ..." + $ADDONS_PATH/$ADDON/disable rm $ADDON_DATA/enabled From de96305a7170407a41a49c3db5d5d00573cdd051 Mon Sep 17 00:00:00 2001 From: Paul Lietar Date: Fri, 29 Nov 2013 13:35:51 +0000 Subject: [PATCH 17/20] Don't fail when trying to renable or reprovision. --- plugins/addons/addons-common.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/plugins/addons/addons-common.sh b/plugins/addons/addons-common.sh index e4a1d01ff3b..8b3c1066cd2 100644 --- a/plugins/addons/addons-common.sh +++ b/plugins/addons/addons-common.sh @@ -7,6 +7,12 @@ function die exit 1 } +function quit +{ + $QUIET || echo $* >&2 + exit 0 +} + function export_addon_vars() { export ADDON="$1" @@ -71,7 +77,7 @@ function check_addon_disabled { check_addon $1 if [ -f $ADDON_DATA/enabled ]; then - die "Add-on $ADDON is already enabled" + quit "Add-on $ADDON is already enabled" fi } @@ -93,7 +99,7 @@ function check_addon_provisioned function check_addon_unprovisioned { if grep -q "^$ADDON:" $ADDONS_FILE; then - die "App $APP already has addon $ADDON" + quit "App $APP already has addon $ADDON" fi } From a892fe701a4b7fe000d5149996434ce50723b01b Mon Sep 17 00:00:00 2001 From: Paul Lietar Date: Fri, 29 Nov 2013 13:36:47 +0000 Subject: [PATCH 18/20] Restart the application once we've added/removed an addon. --- plugins/addons/addons-common.sh | 9 +++++++++ plugins/addons/commands | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/plugins/addons/addons-common.sh b/plugins/addons/addons-common.sh index 8b3c1066cd2..d66a4e11ce6 100644 --- a/plugins/addons/addons-common.sh +++ b/plugins/addons/addons-common.sh @@ -117,3 +117,12 @@ function split_addon_line fi } +function restart_app { + IMAGE="app/$APP" + + echo "-----> Releasing $APP ..." + dokku release $APP $IMAGE + echo "-----> Deploying $APP ..." + dokku deploy $APP $IMAGE +} + diff --git a/plugins/addons/commands b/plugins/addons/commands index c4ab8193c21..197fe27a625 100755 --- a/plugins/addons/commands +++ b/plugins/addons/commands @@ -56,6 +56,8 @@ case "$1" in echo "$ADDON:$id_private" >> $ADDONS_FILE + restart_app + ;; addons:remove) @@ -70,6 +72,8 @@ case "$1" in sed -i "/^$ADDON:/d" $ADDONS_FILE sed "/^export $key=/d" $ENV_FILE | sort > $ENV_FILE + restart_app + ;; addons:enable) From c65e34213c80fc99d31d5efbd28162ce953788a0 Mon Sep 17 00:00:00 2001 From: Paul Lietar Date: Fri, 29 Nov 2013 13:38:37 +0000 Subject: [PATCH 19/20] Make test_deploy more flexible. Allow deploying an application in any path, and allow applications to have a setup script --- tests/test_deploy | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_deploy b/tests/test_deploy index 48626e035da..ca7c4ff4c8f 100755 --- a/tests/test_deploy +++ b/tests/test_deploy @@ -3,7 +3,7 @@ set -eo pipefail SELF=`which $0`; APP="$1"; TARGET="$2"; FORWARDED_PORT="$3" TMP=$(mktemp -d -t "$TARGET.XXXXX") trap "rm -rf $TMP" EXIT -rmdir $TMP && cp -r $(dirname $SELF)/$APP $TMP +rmdir $TMP && cp -r $APP $TMP cd $TMP git init git config user.email "robot@example.com" @@ -14,6 +14,11 @@ REPO="test-$(basename $APP)-$RANDOM" git remote add target dokku@$TARGET:$REPO git push target master URL=$(ssh dokku@$TARGET url $REPO)$FORWARDED_PORT + +if [[ -x setup ]]; then + ./setup $TARGET $REPO +fi + sleep 2 ./check_deploy $URL && echo "-----> Deploy success!" || { sleep 4 From 914d4d15ec4cb92f6ebf2bcb1f9b3ecc7d9e23f6 Mon Sep 17 00:00:00 2001 From: Paul Lietar Date: Sun, 1 Dec 2013 00:31:58 +0000 Subject: [PATCH 20/20] Integrate add ons with the backup plugin. --- plugins/addons/backup-check | 21 ++++++++++++++++ plugins/addons/backup-export | 38 ++++++++++++++++++++++++++++ plugins/addons/backup-import | 49 ++++++++++++++++++++++++++++++++++++ plugins/addons/commands | 27 ++++++++++++++++++++ 4 files changed, 135 insertions(+) create mode 100755 plugins/addons/backup-check create mode 100755 plugins/addons/backup-export create mode 100755 plugins/addons/backup-import diff --git a/plugins/addons/backup-check b/plugins/addons/backup-check new file mode 100755 index 00000000000..2aed4fba804 --- /dev/null +++ b/plugins/addons/backup-check @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x +VERSION="$1" +IMPORT_DIR="$2" +TARGET_DIR="$3" + +QUIET=true +source $(dirname ${BASH_SOURCE[0]})/addons-common.sh + +[[ -f $IMPORT_DIR/.dokku_addons ]] || exit 0 + +ret=0 +while read ADDON; do + if [[ ! -d $ADDONS_PATH/$ADDON ]]; then + ret=1 + echo "Addon $ADDON missing" 2>&1 + fi +done < $IMPORT_DIR/.dokku_addons + +exit $ret + diff --git a/plugins/addons/backup-export b/plugins/addons/backup-export new file mode 100755 index 00000000000..8090a6857d2 --- /dev/null +++ b/plugins/addons/backup-export @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x +VERSION="$1" +BASE_DIR="$2" +TMP_DIR="$3" + +QUIET=true +source $(dirname ${BASH_SOURCE[0]})/addons-common.sh + +cat; + +for addon in $BASE_DIR/.addons/*; do + if [[ -f $addon/enabled ]]; then + echo $(basename $addon) + fi +done > "$TMP_DIR/.dokku_addons" + +echo .dokku_addons + +for appdir in $BASE_DIR/*; do + APP=$(basename $appdir) + if [[ -f $appdir/ADDONS ]]; then + echo $appdir/ADDONS + + while read line; do + split_addon_line $line ADDON ADDON_ID ADDON_PRIVATE + export_addon_vars $ADDON + + mkdir -p "$TMP_DIR/.dokku_addons_dump/$ADDON" + + if [[ -x $ADDON_ROOT/dump ]]; then + $ADDON_ROOT/dump $ADDON_ID $ADDON_PRIVATE | gzip > $TMP_DIR/.dokku_addons_dump/$ADDON/${APP}.gz + echo .dokku_addons_dump/$ADDON/${APP}.gz + fi + done < $appdir/ADDONS + fi +done + diff --git a/plugins/addons/backup-import b/plugins/addons/backup-import new file mode 100755 index 00000000000..393c68ec42b --- /dev/null +++ b/plugins/addons/backup-import @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x +VERSION="$1" +IMPORT_DIR="$2" +TARGET_DIR="$3" + +BACKUP_TMP_DIR="$2/../../" # TODO: Make this less hacky + +QUIET=true +source $(dirname ${BASH_SOURCE[0]})/addons-common.sh + +if [[ ! -f $BACKUP_TMP_DIR/.dokku_addons ]]; then + exit 0 +fi + +while read ADDON; do + mkdir -p $DOKKU_ROOT/.addons/$ADDON + export_addon_vars $ADDON + + if [[ ! -f $ADDON_DATA/enabled ]]; then + $ADDON_ROOT/enable + touch $ADDON_DATA/enabled + fi +done < $BACKUP_TMP_DIR/.dokku_addons + +sleep 5 # Wait for freshly enabled addons to start + +for appdir in $IMPORT_DIR/*; do + APP=$(basename $appdir) + + if [[ -f $appdir/ADDONS ]]; then + while read line; do + split_addon_line $line ADDON ADDON_ID ADDON_PRIVATE + export_addon_vars $ADDON + + id_private=$($ADDON_ROOT/provision $APP) + line="$ADDON:$id_private" + echo $line >> $TARGET_DIR/$APP/ADDONS + split_addon_line $line ADDON ADDON_ID ADDON_PRIVATE + + dump=$BACKUP_TMP_DIR/.dokku_addons_dump/$ADDON/${APP}.gz + + if [[ -f $dump && -x $ADDON_ROOT/restore ]]; then + zcat $dump | $ADDON_ROOT/restore $ADDON_ID $ADDON_PRIVATE + fi + done < $appdir/ADDONS + fi +done + diff --git a/plugins/addons/commands b/plugins/addons/commands index 197fe27a625..6bb5db3a73f 100755 --- a/plugins/addons/commands +++ b/plugins/addons/commands @@ -114,6 +114,33 @@ case "$1" in ;; + addons:dump) + check_app $2 + check_addon $3 + check_addon_provisioned + + if [[ -x $ADDON_ROOT/dump ]]; then + $ADDON_ROOT/dump $ADDON_ID $ADDON_PRIVATE + else + die "Addon $ADDON does not support dumping" + fi + + ;; + + addons:restore) + check_app $2 + check_addon $3 + check_addon_provisioned + + if [[ -x $ADDON_ROOT/dump ]]; then + $ADDON_ROOT/restore $ADDON_ID $ADDON_PRIVATE + else + die "Addon $ADDON does not support restoring" + fi + + ;; + + help) cat && cat<