diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed0f7c8aa78..676b34324c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,6 +66,9 @@ jobs: - name: run ci timeout-minutes: 30 run: sudo -E ./.github/commands/ci-run ${{ matrix.index }} + env: + SYNC_GITHUB_PASSWORD: ${{ secrets.SYNC_GITHUB_PASSWORD }} + SYNC_GITHUB_USERNAME: ${{ secrets.SYNC_GITHUB_USERNAME }} - uses: actions/upload-artifact@v2 with: diff --git a/Makefile b/Makefile index 417c7fef9b6..b7914dee618 100644 --- a/Makefile +++ b/Makefile @@ -2,11 +2,13 @@ DOKKU_VERSION ?= master DOCKER_IMAGE_LABELER_VERSION ?= 0.2.2 HEROKUISH_VERSION ?= 0.5.25 +NETRC_VERSION ?= 0.3.0 PLUGN_VERSION ?= 0.6.1 PROCFILE_VERSION ?= 0.11.0 SIGIL_VERSION ?= 0.6.0 SSHCOMMAND_VERSION ?= 0.12.0 DOCKER_IMAGE_LABELER_URL ?= https://github.com/dokku/docker-image-labeler/releases/download/v${DOCKER_IMAGE_LABELER_VERSION}/docker-image-labeler_${DOCKER_IMAGE_LABELER_VERSION}_linux_x86_64.tgz +NETRC_URL ?= https://github.com/dokku/netrc/releases/download/v${NETRC_VERSION}/netrc_${NETRC_VERSION}_linux_x86_64.tgz PLUGN_URL ?= https://github.com/dokku/plugn/releases/download/v${PLUGN_VERSION}/plugn_${PLUGN_VERSION}_linux_x86_64.tgz PROCFILE_UTIL_URL ?= https://github.com/josegonzalez/go-procfile-util/releases/download/v${PROCFILE_VERSION}/procfile-util_${PROCFILE_VERSION}_linux_x86_64.tgz SIGIL_URL ?= https://github.com/gliderlabs/sigil/releases/download/v${SIGIL_VERSION}/sigil_${SIGIL_VERSION}_Linux_x86_64.tgz @@ -34,7 +36,7 @@ endif include common.mk -.PHONY: all apt-update install version copyfiles copyplugin man-db plugins dependencies docker-image-labeler sshcommand procfile-util plugn docker aufs stack count dokku-installer vagrant-acl-add vagrant-dokku go-build +.PHONY: all apt-update install version copyfiles copyplugin man-db plugins dependencies docker-image-labeler netrc sshcommand procfile-util plugn docker aufs stack count dokku-installer vagrant-acl-add vagrant-dokku go-build include tests.mk include package.mk @@ -125,7 +127,7 @@ plugin-dependencies: plugn procfile-util plugins: plugn procfile-util docker sudo -E dokku plugin:install --core -dependencies: apt-update docker-image-labeler sshcommand plugn procfile-util docker help2man man-db sigil dos2unix jq +dependencies: apt-update docker-image-labeler netrc sshcommand plugn procfile-util docker help2man man-db sigil dos2unix jq $(MAKE) -e stack apt-update: @@ -147,10 +149,9 @@ docker-image-labeler: wget -qO /tmp/docker-image-labeler_latest.tgz ${DOCKER_IMAGE_LABELER_URL} tar xzf /tmp/docker-image-labeler_latest.tgz -C /usr/local/bin -sshcommand: - wget -qO /tmp/sshcommand_latest.tgz ${SSHCOMMAND_URL} - tar xzf /tmp/sshcommand_latest.tgz -C /usr/local/bin - sshcommand create dokku /usr/local/bin/dokku +netrc: + wget -qO /tmp/netrc_latest.tgz ${NETRC_URL} + tar xzf /tmp/netrc_latest.tgz -C /usr/local/bin procfile-util: wget -qO /tmp/procfile-util_latest.tgz ${PROCFILE_UTIL_URL} @@ -164,6 +165,11 @@ sigil: wget -qO /tmp/sigil_latest.tgz ${SIGIL_URL} tar xzf /tmp/sigil_latest.tgz -C /usr/local/bin +sshcommand: + wget -qO /tmp/sshcommand_latest.tgz ${SSHCOMMAND_URL} + tar xzf /tmp/sshcommand_latest.tgz -C /usr/local/bin + sshcommand create dokku /usr/local/bin/dokku + docker: apt-get -qq -y --no-install-recommends install curl grep -i -E "^docker" /etc/group || groupadd docker diff --git a/debian/control b/debian/control index 3f0cad3300b..07e87fb74c0 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Version: 0.23.9 Section: web Priority: optional Architecture: amd64 -Depends: locales, git, cpio, curl, man-db, netcat, sshcommand (>= 0.12.0), docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, docker-image-labeler (>= 0.2.2), net-tools, software-properties-common, procfile-util (>= 0.11.0), python-software-properties | python3-software-properties, rsyslog, dos2unix, jq +Depends: locales, git, cpio, curl, man-db, netcat, sshcommand (>= 0.12.0), docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, docker-image-labeler (>= 0.2.2), net-tools, netrc, software-properties-common, procfile-util (>= 0.11.0), python-software-properties | python3-software-properties, rsyslog, dos2unix, jq Recommends: herokuish (>= 0.3.4), parallel, dokku-update, dokku-event-listener Pre-Depends: gliderlabs-sigil, nginx (>= 1.8.0) | openresty, dnsutils, cgroupfs-mount | cgroup-lite, plugn (>= 0.3.0), sudo, python3, debconf Maintainer: Jose Diaz-Gonzalez diff --git a/docs/deployment/methods/git.md b/docs/deployment/methods/git.md index df47f53fbcc..fc5626cfb3e 100644 --- a/docs/deployment/methods/git.md +++ b/docs/deployment/methods/git.md @@ -4,6 +4,7 @@ ``` git:allow-host # Adds a host to known_hosts +git:auth [ ] # Configures netrc authentication for a given git server git:sync [--build] [] # Clone or fetch an app from remote git repo git:initialize # Initialize a git repository for an app git:public-key # Outputs the dokku public deploy key @@ -115,10 +116,12 @@ Please keep in mind that setting `keep-git-dir` to `true` may result in unstaged ### Initializing an app repository from a remote repository -> The application must exist before the repository can be initialized +> New as of 0.23.0 A Dokku app repository can be initialized or updated from a remote git repository via the `git:sync` command. This command will either clone or fetch updates from a remote repository and has undefined behavior if the history cannot be fast-fowarded to the referenced repository reference. Any repository that can be cloned by the `dokku` user can be specified. +> The application must exist before the repository can be initialized + ```shell dokku git:sync node-js-app https://github.com/heroku/node-js-getting-started.git ``` @@ -142,6 +145,27 @@ By default, this command does not trigger an application build. To do so during dokku git:sync --build node-js-app https://github.com/heroku/node-js-getting-started.git ``` +### Initializing from private repositories + +> New as of 0.24.0 + +Initializing from a private repository requires one of the following: + +- A Public SSH Key (`id_rsa.pub` file) configured on the remote server, with the associated private key (`id_rsa`) in the Dokku server's `/home/dokku/.ssh/` directory. +- A configured [`.netrc`](https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html) entry. + +Dokku provides the `git:auth` command which can be used to configure a `netrc` entry for the remote server. This command can be used to add or remove configuration for any remote server. + +```shell +# add credentials for github.com +dokku git:auth github.com username personal-access-token + +# remove credentials for github.com +dokku git:auth github.com +``` + +For syncing to a private repository stored on a remote Git product such as Github or Gitlab, Dokku's recommendation is to use a personal access token on a bot user where possible. Please see your service's documentation for information regarding the recommended best practices. + ### Allowing remote repository hosts By default, the Dokku host may not have access to a server containing the remote repository. This can be initialized via the `git:allow-host` command. @@ -159,3 +183,5 @@ In order to clone a remote repository, the remote server should have the Dokku h ```shell dokku git:public-key ``` + +If there is no key, an error message is shown that displays the command that can be run on the Dokku server to generate a new public/private ssh key pair. diff --git a/plugins/git/help-functions b/plugins/git/help-functions index 35841f06215..f007b1146e4 100755 --- a/plugins/git/help-functions +++ b/plugins/git/help-functions @@ -28,6 +28,7 @@ fn-help-content() { declare desc="return help content" cat <, Adds a host to known_hosts + git:auth [ ], Configures netrc authentication for a given git server git:sync [--build] [], Clone or fetch an app from remote git repo git:initialize , Initialize a git repository for an app git:public-key, Outputs the dokku public deploy key diff --git a/plugins/git/internal-functions b/plugins/git/internal-functions index 5dd4d5a27e9..0c956dfba5e 100755 --- a/plugins/git/internal-functions +++ b/plugins/git/internal-functions @@ -15,6 +15,27 @@ cmd-git-allow-host() { dokku_log_info1 "$HOST added to known hosts" } +cmd-git-auth() { + declare desc="configures netrc authentication for a given git server" + local cmd="git:auth" + [[ "$1" == "$cmd" ]] && shift 1 + declare HOST="$1" USERNAME="$2" PASSWORD="$3" + [[ -z "$HOST" ]] && dokku_log_fail "Please supply a git host" + + if [[ -n "$USERNAME" ]] && [[ -z "$PASSWORD" ]]; then + dokku_log_fail "Missing password for netrc auth entry" + fi + + if [[ -z "$USERNAME" ]]; then + dokku_log_info1 "Removing netrc auth entry for host $HOST" + netrc unset "$HOST" + return $? + fi + + dokku_log_info1 "Setting netrc auth entry for host $HOST" + netrc set "$HOST" "$USERNAME" "$PASSWORD" +} + cmd-git-sync() { declare desc="clone or fetch an app from remote git repo" local cmd="git:sync" @@ -65,8 +86,9 @@ cmd-git-public-key() { [[ "$1" == "$cmd" ]] && shift 1 if [[ ! -f "$DOKKU_ROOT/.ssh/id_rsa.pub" ]]; then - dokku_log_fail "There is no deploy key" + fn-git-auth-error fi + cat "$DOKKU_ROOT/.ssh/id_rsa.pub" } @@ -137,6 +159,13 @@ cmd-git-report-single() { fi } +fn-git-auth-error() { + dokku_log_warn "There is no deploy key associated with the $DOKKU_SYSTEM_USER user." + dokku_log_warn "Generate an ssh key with the following command (do not specify a password):" + dokku_log_warn " ssh-keygen -t ed25519 -C 'example@example.com'" + dokku_log_fail "As an alternative, configure the netrc authentication via the git:auth command" +} + fn-git-create-hook() { declare APP="$1" local APP_PATH="$DOKKU_ROOT/$APP" @@ -179,14 +208,16 @@ fn-git-clone() { trap 'rm -rf $APP_CLONE_ROOT > /dev/null' RETURN INT TERM EXIT is_ref=true - git clone --depth 1 -n -qq --branch "$GIT_REF" "$GIT_REMOTE" "$APP_CLONE_ROOT" 2>/dev/null && is_ref=false + if GIT_TERMINAL_PROMPT=0 git clone --depth 1 -n --branch "$GIT_REF" "$GIT_REMOTE" "$APP_CLONE_ROOT" 2>/dev/null; then + is_ref=false + fi rm -rf "$APP_CLONE_ROOT" if [[ "$is_ref" == "true" ]]; then - suppress_output git clone -n -qq "$GIT_REMOTE" "$APP_CLONE_ROOT" + GIT_TERMINAL_PROMPT=0 suppress_output git clone -n "$GIT_REMOTE" "$APP_CLONE_ROOT" fn-git-cmd "$APP_CLONE_ROOT" checkout -qq "$GIT_REF" else - suppress_output git clone -n -qq --branch "$GIT_REF" "$GIT_REMOTE" "$APP_CLONE_ROOT" + GIT_TERMINAL_PROMPT=0 suppress_output git clone -n --branch "$GIT_REF" "$GIT_REMOTE" "$APP_CLONE_ROOT" if fn-git-cmd "$APP_CLONE_ROOT" show-ref --verify "refs/heads/$GIT_REF" >/dev/null 2>&1; then dokku_log_verbose "Detected branch, setting deploy-branch to $GIT_REF" fn-plugin-property-write "git" "$APP" "deploy-branch" "$GIT_REF" @@ -229,17 +260,17 @@ fn-git-fetch() { DOKKU_DEPLOY_BRANCH="$(fn-git-deploy-branch "$APP")" if ! fn-git-cmd "$APP_ROOT" check-ref-format --branch "$DOKKU_DEPLOY_BRANCH" >/dev/null 2>&1; then - echo $'\e[1G\e[K'"-----> WARNING: Invalid branch name '$DOKKU_DEPLOY_BRANCH' specified via DOKKU_DEPLOY_BRANCH." - echo $'\e[1G\e[K'"-----> For more details, please see the man page for 'git-check-ref-format.'" + dokku_log_warn "Invalid branch name '$DOKKU_DEPLOY_BRANCH' specified via DOKKU_DEPLOY_BRANCH." + dokku_log_warn "For more details, please see the man page for 'git-check-ref-format.'" return fi fn-git-cmd "$APP_ROOT" remote rm remote >/dev/null 2>&1 || true fn-git-cmd "$APP_ROOT" remote add --mirror=fetch --no-tags remote "$GIT_REMOTE" if [[ -z "$GIT_REF" ]]; then - fn-git-cmd "$APP_ROOT" fetch --update-head-ok remote "$DOKKU_DEPLOY_BRANCH" + GIT_TERMINAL_PROMPT=0 fn-git-cmd "$APP_ROOT" fetch --update-head-ok remote "$DOKKU_DEPLOY_BRANCH" else - fn-git-cmd "$APP_ROOT" fetch --update-head-ok remote + GIT_TERMINAL_PROMPT=0 fn-git-cmd "$APP_ROOT" fetch --update-head-ok remote fn-git-cmd "$APP_ROOT" update-ref "refs/heads/$DOKKU_DEPLOY_BRANCH" "$GIT_REF" fi } diff --git a/plugins/git/subcommands/auth b/plugins/git/subcommands/auth new file mode 100755 index 00000000000..f8d31d10a34 --- /dev/null +++ b/plugins/git/subcommands/auth @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +source "$PLUGIN_AVAILABLE_PATH/git/internal-functions" +set -eo pipefail +[[ $DOKKU_TRACE ]] && set -x + +cmd-git-auth "$@" diff --git a/rpm.mk b/rpm.mk index be3d6335e44..4b7bf6b10bf 100644 --- a/rpm.mk +++ b/rpm.mk @@ -40,6 +40,7 @@ endif --depends 'jq' \ --depends 'man-db' \ --depends 'nc' \ + --depends 'netrc' \ --depends 'nginx >= 1.8.0' \ --depends 'plugn' \ --depends 'procfile-util >= 0.11.0' \ diff --git a/tests/ci/setup.sh b/tests/ci/setup.sh index b89e8b67c0a..716e3431aca 100755 --- a/tests/ci/setup.sh +++ b/tests/ci/setup.sh @@ -20,6 +20,12 @@ install_dependencies() { curl -L "https://packagecloud.io/dokku/dokku/packages/ubuntu/bionic/herokuish_${HEROKUISH_VERSION}_amd64.deb/download.deb" -o "$ROOT_DIR/build/${HEROKUISH_PACKAGE_NAME}" fi + NETRC_VERSION=$(grep NETRC_VERSION "${ROOT_DIR}/Makefile" | head -n1 | cut -d' ' -f3) + NETRC_PACKAGE_NAME="netrc_${NETRC_VERSION}_amd64.deb" + if [[ ! -f "$ROOT_DIR/build/${NETRC_PACKAGE_NAME}" ]]; then + curl -L "https://packagecloud.io/dokku/dokku/packages/ubuntu/bionic/netrc_${NETRC_VERSION}_amd64.deb/download.deb" -o "$ROOT_DIR/build/${NETRC_PACKAGE_NAME}" + fi + PLUGN_VERSION=$(grep PLUGN_VERSION "${ROOT_DIR}/Makefile" | head -n1 | cut -d' ' -f3) PLUGN_PACKAGE_NAME="plugn_${PLUGN_VERSION}_amd64.deb" if [[ ! -f "$ROOT_DIR/build/${PLUGN_PACKAGE_NAME}" ]]; then @@ -52,6 +58,7 @@ install_dependencies() { sudo dpkg -i \ "${ROOT_DIR}/build/$DOCKER_IMAGE_LABELER_PACKAGE_NAME" \ "${ROOT_DIR}/build/$HEROKUISH_PACKAGE_NAME" \ + "${ROOT_DIR}/build/$NETRC_PACKAGE_NAME" \ "${ROOT_DIR}/build/$PLUGN_PACKAGE_NAME" \ "${ROOT_DIR}/build/$PROCFILE_UTIL_PACKAGE_NAME" \ "${ROOT_DIR}/build/$SIGIL_PACKAGE_NAME" \ diff --git a/tests/unit/git_3.bats b/tests/unit/git_3.bats index 7586b0b5af4..392e8f8d4a2 100644 --- a/tests/unit/git_3.bats +++ b/tests/unit/git_3.bats @@ -5,12 +5,16 @@ load test_helper setup() { global_setup create_app + [[ -f "$DOKKU_ROOT/.netrc" ]] && cp -fp "$DOKKU_ROOT/.netrc" "$DOKKU_ROOT/.netrc.bak" + touch /home/dokku/.netrc + chown dokku:dokku /home/dokku/.netrc touch /home/dokku/.ssh/known_hosts chown dokku:dokku /home/dokku/.ssh/known_hosts } teardown() { rm -f /home/dokku/.ssh/id_rsa.pub || true + [[ -f "$DOKKU_ROOT/.netrc.bak" ]] && mv "$DOKKU_ROOT/.netrc.bak" "$DOKKU_ROOT/.netrc" && chown dokku:dokku "$DOKKU_ROOT/.netrc" destroy_app global_teardown } @@ -60,6 +64,31 @@ teardown() { assert_equal "$output" "$((start_lines + 2))" } +@test "(git) git:auth" { + run /bin/bash -c "dokku git:auth" + echo "output: $output" + echo "status: $status" + assert_failure + + run /bin/bash -c "dokku git:auth github.com" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "Removing netrc auth entry for host github.com" + + run /bin/bash -c "dokku git:auth github.com username" + echo "output: $output" + echo "status: $status" + assert_failure + assert_output_contains "Missing password for netrc auth entry" + + run /bin/bash -c "dokku git:auth github.com username password" + echo "output: $output" + echo "status: $status" + assert_success + assert_output_contains "Setting netrc auth entry for host github.com" +} + @test "(git) git:sync new [errors]" { run /bin/bash -c "dokku git:sync" echo "output: $output" @@ -274,11 +303,40 @@ teardown() { assert_output_contains "Application deployed" } + +@test "(git) git:sync private" { + if [[ -z "$SYNC_GITHUB_USERNAME" ]] || [[ -z "$SYNC_GITHUB_PASSWORD" ]]; then + skip "skipping due to missing github credentials SYNC_GITHUB_USERNAME:SYNC_GITHUB_PASSWORD" + fi + + run /bin/bash -c "dokku git:sync $TEST_APP https://github.com/dokku/smoke-test-app-private.git" + echo "output: $output" + echo "status: $status" + assert_failure + assert_output_contains "fatal: could not read Username for" + + run /bin/bash -c "dokku git:auth github.com $SYNC_GITHUB_USERNAME $SYNC_GITHUB_PASSWORD" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "cat /home/dokku/.netrc" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku git:sync $TEST_APP https://github.com/dokku/smoke-test-app-private.git" + echo "output: $output" + echo "status: $status" + assert_success +} + @test "(git) git:public-key" { run /bin/bash -c "dokku git:public-key" echo "output: $output" echo "status: $status" assert_failure + assert_output_contains "There is no deploy key associated with the dokku user." run /bin/bash -c "cp /root/.ssh/dokku_test_rsa.pub /home/dokku/.ssh/id_rsa.pub" echo "output: $output"