dokku · josegonzalez · Feb 28, 2021 · Feb 21, 2021
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -66,6 +66,9 @@ jobs:
       - name: run ci
         timeout-minutes: 30
         run: sudo -E ./.github/commands/ci-run ${{ matrix.index }}
+        env:
+          SYNC_GITHUB_PASSWORD: ${{ secrets.SYNC_GITHUB_PASSWORD }}
+          SYNC_GITHUB_USERNAME: ${{ secrets.SYNC_GITHUB_USERNAME }}
 
       - uses: actions/upload-artifact@v2
         with:

diff --git a/Makefile b/Makefile
@@ -2,11 +2,13 @@ DOKKU_VERSION ?= master
 
 DOCKER_IMAGE_LABELER_VERSION ?= 0.2.2
 HEROKUISH_VERSION ?= 0.5.25
+NETRC_VERSION ?= 0.3.0
 PLUGN_VERSION ?= 0.6.1
 PROCFILE_VERSION ?= 0.11.0
 SIGIL_VERSION ?= 0.6.0
 SSHCOMMAND_VERSION ?= 0.12.0
 DOCKER_IMAGE_LABELER_URL ?= https://github.com/dokku/docker-image-labeler/releases/download/v${DOCKER_IMAGE_LABELER_VERSION}/docker-image-labeler_${DOCKER_IMAGE_LABELER_VERSION}_linux_x86_64.tgz
+NETRC_URL ?= https://github.com/dokku/netrc/releases/download/v${NETRC_VERSION}/netrc_${NETRC_VERSION}_linux_x86_64.tgz
 PLUGN_URL ?= https://github.com/dokku/plugn/releases/download/v${PLUGN_VERSION}/plugn_${PLUGN_VERSION}_linux_x86_64.tgz
 PROCFILE_UTIL_URL ?= https://github.com/josegonzalez/go-procfile-util/releases/download/v${PROCFILE_VERSION}/procfile-util_${PROCFILE_VERSION}_linux_x86_64.tgz
 SIGIL_URL ?= https://github.com/gliderlabs/sigil/releases/download/v${SIGIL_VERSION}/sigil_${SIGIL_VERSION}_Linux_x86_64.tgz
@@ -34,7 +36,7 @@ endif
 
 include common.mk
 
-.PHONY: all apt-update install version copyfiles copyplugin man-db plugins dependencies docker-image-labeler sshcommand procfile-util plugn docker aufs stack count dokku-installer vagrant-acl-add vagrant-dokku go-build
+.PHONY: all apt-update install version copyfiles copyplugin man-db plugins dependencies docker-image-labeler netrc sshcommand procfile-util plugn docker aufs stack count dokku-installer vagrant-acl-add vagrant-dokku go-build
 
 include tests.mk
 include package.mk
@@ -125,7 +127,7 @@ plugin-dependencies: plugn procfile-util
 plugins: plugn procfile-util docker
 	sudo -E dokku plugin:install --core
 
-dependencies: apt-update docker-image-labeler sshcommand plugn procfile-util docker help2man man-db sigil dos2unix jq
+dependencies: apt-update docker-image-labeler netrc sshcommand plugn procfile-util docker help2man man-db sigil dos2unix jq
 	$(MAKE) -e stack
 
 apt-update:
@@ -147,10 +149,9 @@ docker-image-labeler:
 	wget -qO /tmp/docker-image-labeler_latest.tgz ${DOCKER_IMAGE_LABELER_URL}
 	tar xzf /tmp/docker-image-labeler_latest.tgz -C /usr/local/bin
 
-sshcommand:
-	wget -qO /tmp/sshcommand_latest.tgz ${SSHCOMMAND_URL}
-	tar xzf /tmp/sshcommand_latest.tgz -C /usr/local/bin
-	sshcommand create dokku /usr/local/bin/dokku
+netrc:
+	wget -qO /tmp/netrc_latest.tgz ${NETRC_URL}
+	tar xzf /tmp/netrc_latest.tgz -C /usr/local/bin
 
 procfile-util:
 	wget -qO /tmp/procfile-util_latest.tgz ${PROCFILE_UTIL_URL}
@@ -164,6 +165,11 @@ sigil:
 	wget -qO /tmp/sigil_latest.tgz ${SIGIL_URL}
 	tar xzf /tmp/sigil_latest.tgz -C /usr/local/bin
 
+sshcommand:
+	wget -qO /tmp/sshcommand_latest.tgz ${SSHCOMMAND_URL}
+	tar xzf /tmp/sshcommand_latest.tgz -C /usr/local/bin
+	sshcommand create dokku /usr/local/bin/dokku
+
 docker:
 	apt-get -qq -y --no-install-recommends install curl
 	grep -i -E "^docker" /etc/group || groupadd docker

diff --git a/debian/control b/debian/control
@@ -3,7 +3,7 @@ Version: 0.23.9
 Section: web
 Priority: optional
 Architecture: amd64
-Depends: locales, git, cpio, curl, man-db, netcat, sshcommand (>= 0.12.0), docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, docker-image-labeler (>= 0.2.2), net-tools, software-properties-common, procfile-util (>= 0.11.0), python-software-properties | python3-software-properties, rsyslog, dos2unix, jq
+Depends: locales, git, cpio, curl, man-db, netcat, sshcommand (>= 0.12.0), docker-engine-cs (>= 17.05.0) | docker-engine (>= 17.05.0) | docker-io (>= 17.05.0) | docker.io (>= 17.05.0) | docker-ce (>= 17.05.0) | docker-ee (>= 17.05.0) | moby-engine, docker-image-labeler (>= 0.2.2), net-tools, netrc, software-properties-common, procfile-util (>= 0.11.0), python-software-properties | python3-software-properties, rsyslog, dos2unix, jq
 Recommends: herokuish (>= 0.3.4), parallel, dokku-update, dokku-event-listener
 Pre-Depends: gliderlabs-sigil, nginx (>= 1.8.0) | openresty, dnsutils, cgroupfs-mount | cgroup-lite, plugn (>= 0.3.0), sudo, python3, debconf
 Maintainer: Jose Diaz-Gonzalez <dokku@josediazgonzalez.com>

diff --git a/docs/deployment/methods/git.md b/docs/deployment/methods/git.md
@@ -4,6 +4,7 @@
 
 ```
 git:allow-host <host>                              # Adds a host to known_hosts
+git:auth <host> [<username> <password>]            # Configures netrc authentication for a given git server
 git:sync [--build] <app> <repository> [<git-ref>]  # Clone or fetch an app from remote git repo
 git:initialize <app>                               # Initialize a git repository for an app
 git:public-key                                     # Outputs the dokku public deploy key
@@ -115,10 +116,12 @@ Please keep in mind that setting `keep-git-dir` to `true` may result in unstaged
 
 ### Initializing an app repository from a remote repository
 
-> The application must exist before the repository can be initialized
+> New as of 0.23.0
 
 A Dokku app repository can be initialized or updated from a remote git repository via the `git:sync` command. This command will either clone or fetch updates from a remote repository and has undefined behavior if the history cannot be fast-fowarded to the referenced repository reference. Any repository that can be cloned by the `dokku` user can be specified.
 
+> The application must exist before the repository can be initialized
+
 ```shell
 dokku git:sync node-js-app https://github.com/heroku/node-js-getting-started.git
 ```
@@ -142,6 +145,27 @@ By default, this command does not trigger an application build. To do so during
 dokku git:sync --build node-js-app https://github.com/heroku/node-js-getting-started.git
 ```
 
+### Initializing from private repositories
+
+> New as of 0.24.0
+
+Initializing from a private repository requires one of the following:
+
+- A Public SSH Key (`id_rsa.pub` file) configured on the remote server, with the associated private key (`id_rsa`) in the Dokku server's `/home/dokku/.ssh/` directory.
+- A configured [`.netrc`](https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html) entry.
+
+Dokku provides the `git:auth` command which can be used to configure a `netrc` entry for the remote server. This command can be used to add or remove configuration for any remote server.
+
+```shell
+# add credentials for github.com
+dokku git:auth github.com username personal-access-token
+
+# remove credentials for github.com
+dokku git:auth github.com
+```
+
+For syncing to a private repository stored on a remote Git product such as Github or Gitlab, Dokku's recommendation is to use a personal access token on a bot user where possible. Please see your service's documentation for information regarding the recommended best practices.
+
 ### Allowing remote repository hosts
 
 By default, the Dokku host may not have access to a server containing the remote repository. This can be initialized via the `git:allow-host` command.
@@ -159,3 +183,5 @@ In order to clone a remote repository, the remote server should have the Dokku h
 ```shell
 dokku git:public-key
 ```
+
+If there is no key, an error message is shown that displays the command that can be run on the Dokku server to generate a new public/private ssh key pair.
diff --git a/plugins/git/help-functions b/plugins/git/help-functions
@@ -28,6 +28,7 @@ fn-help-content() {
   declare desc="return help content"
   cat <<help_content
     git:allow-host <host>, Adds a host to known_hosts
+    git:auth <host> [<username> <password>], Configures netrc authentication for a given git server
     git:sync [--build] <app> <repository> [<git-ref>], Clone or fetch an app from remote git repo
     git:initialize <app>, Initialize a git repository for an app
     git:public-key, Outputs the dokku public deploy key

diff --git a/plugins/git/internal-functions b/plugins/git/internal-functions
@@ -15,6 +15,27 @@ cmd-git-allow-host() {
   dokku_log_info1 "$HOST added to known hosts"
 }
 
+cmd-git-auth() {
+  declare desc="configures netrc authentication for a given git server"
+  local cmd="git:auth"
+  [[ "$1" == "$cmd" ]] && shift 1
+  declare HOST="$1" USERNAME="$2" PASSWORD="$3"
+  [[ -z "$HOST" ]] && dokku_log_fail "Please supply a git host"
+
+  if [[ -n "$USERNAME" ]] && [[ -z "$PASSWORD" ]]; then
+    dokku_log_fail "Missing password for netrc auth entry"
+  fi
+
+  if [[ -z "$USERNAME" ]]; then
+    dokku_log_info1 "Removing netrc auth entry for host $HOST"
+    netrc unset "$HOST"
+    return $?
+  fi
+
+  dokku_log_info1 "Setting netrc auth entry for host $HOST"
+  netrc set "$HOST" "$USERNAME" "$PASSWORD"
+}
+
 cmd-git-sync() {
   declare desc="clone or fetch an app from remote git repo"
   local cmd="git:sync"
@@ -65,8 +86,9 @@ cmd-git-public-key() {
   [[ "$1" == "$cmd" ]] && shift 1
 
   if [[ ! -f "$DOKKU_ROOT/.ssh/id_rsa.pub" ]]; then
-    dokku_log_fail "There is no deploy key"
+    fn-git-auth-error
   fi
+
   cat "$DOKKU_ROOT/.ssh/id_rsa.pub"
 }
 
@@ -137,6 +159,13 @@ cmd-git-report-single() {
   fi
 }
 
+fn-git-auth-error() {
+  dokku_log_warn "There is no deploy key associated with the $DOKKU_SYSTEM_USER user."
+  dokku_log_warn "Generate an ssh key with the following command (do not specify a password):"
+  dokku_log_warn "  ssh-keygen -t ed25519 -C 'example@example.com'"
+  dokku_log_fail "As an alternative, configure the netrc authentication via the git:auth command"
+}
+
 fn-git-create-hook() {
   declare APP="$1"
   local APP_PATH="$DOKKU_ROOT/$APP"
@@ -179,14 +208,16 @@ fn-git-clone() {
   trap 'rm -rf $APP_CLONE_ROOT > /dev/null' RETURN INT TERM EXIT
 
   is_ref=true
-  git clone --depth 1 -n -qq --branch "$GIT_REF" "$GIT_REMOTE" "$APP_CLONE_ROOT" 2>/dev/null && is_ref=false
+  if GIT_TERMINAL_PROMPT=0 git clone --depth 1 -n --branch "$GIT_REF" "$GIT_REMOTE" "$APP_CLONE_ROOT" 2>/dev/null; then
+    is_ref=false
+  fi
   rm -rf "$APP_CLONE_ROOT"
 
   if [[ "$is_ref" == "true" ]]; then
-    suppress_output git clone -n -qq "$GIT_REMOTE" "$APP_CLONE_ROOT"
+    GIT_TERMINAL_PROMPT=0 suppress_output git clone -n "$GIT_REMOTE" "$APP_CLONE_ROOT"
     fn-git-cmd "$APP_CLONE_ROOT" checkout -qq "$GIT_REF"
   else
-    suppress_output git clone -n -qq --branch "$GIT_REF" "$GIT_REMOTE" "$APP_CLONE_ROOT"
+    GIT_TERMINAL_PROMPT=0 suppress_output git clone -n --branch "$GIT_REF" "$GIT_REMOTE" "$APP_CLONE_ROOT"
     if fn-git-cmd "$APP_CLONE_ROOT" show-ref --verify "refs/heads/$GIT_REF" >/dev/null 2>&1; then
       dokku_log_verbose "Detected branch, setting deploy-branch to $GIT_REF"
       fn-plugin-property-write "git" "$APP" "deploy-branch" "$GIT_REF"
@@ -229,17 +260,17 @@ fn-git-fetch() {
 
   DOKKU_DEPLOY_BRANCH="$(fn-git-deploy-branch "$APP")"
   if ! fn-git-cmd "$APP_ROOT" check-ref-format --branch "$DOKKU_DEPLOY_BRANCH" >/dev/null 2>&1; then
-    echo $'\e[1G\e[K'"-----> WARNING: Invalid branch name '$DOKKU_DEPLOY_BRANCH' specified via DOKKU_DEPLOY_BRANCH."
-    echo $'\e[1G\e[K'"-----> For more details, please see the man page for 'git-check-ref-format.'"
+    dokku_log_warn "Invalid branch name '$DOKKU_DEPLOY_BRANCH' specified via DOKKU_DEPLOY_BRANCH."
+    dokku_log_warn "For more details, please see the man page for 'git-check-ref-format.'"
     return
   fi
 
   fn-git-cmd "$APP_ROOT" remote rm remote >/dev/null 2>&1 || true
   fn-git-cmd "$APP_ROOT" remote add --mirror=fetch --no-tags remote "$GIT_REMOTE"
   if [[ -z "$GIT_REF" ]]; then
-    fn-git-cmd "$APP_ROOT" fetch --update-head-ok remote "$DOKKU_DEPLOY_BRANCH"
+    GIT_TERMINAL_PROMPT=0 fn-git-cmd "$APP_ROOT" fetch --update-head-ok remote "$DOKKU_DEPLOY_BRANCH"
   else
-    fn-git-cmd "$APP_ROOT" fetch --update-head-ok remote
+    GIT_TERMINAL_PROMPT=0 fn-git-cmd "$APP_ROOT" fetch --update-head-ok remote
     fn-git-cmd "$APP_ROOT" update-ref "refs/heads/$DOKKU_DEPLOY_BRANCH" "$GIT_REF"
   fi
 }

diff --git a/plugins/git/subcommands/auth b/plugins/git/subcommands/auth
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+source "$PLUGIN_AVAILABLE_PATH/git/internal-functions"
+set -eo pipefail
+[[ $DOKKU_TRACE ]] && set -x
+
+cmd-git-auth "$@"
diff --git a/rpm.mk b/rpm.mk
@@ -40,6 +40,7 @@ endif
 		--depends 'jq' \
 		--depends 'man-db' \
 		--depends 'nc' \
+		--depends 'netrc' \
 		--depends 'nginx >= 1.8.0' \
 		--depends 'plugn' \
 		--depends 'procfile-util >= 0.11.0' \

diff --git a/tests/ci/setup.sh b/tests/ci/setup.sh
@@ -20,6 +20,12 @@ install_dependencies() {
     curl -L "https://packagecloud.io/dokku/dokku/packages/ubuntu/bionic/herokuish_${HEROKUISH_VERSION}_amd64.deb/download.deb" -o "$ROOT_DIR/build/${HEROKUISH_PACKAGE_NAME}"
   fi
 
+  NETRC_VERSION=$(grep NETRC_VERSION "${ROOT_DIR}/Makefile" | head -n1 | cut -d' ' -f3)
+  NETRC_PACKAGE_NAME="netrc_${NETRC_VERSION}_amd64.deb"
+  if [[ ! -f "$ROOT_DIR/build/${NETRC_PACKAGE_NAME}" ]]; then
+    curl -L "https://packagecloud.io/dokku/dokku/packages/ubuntu/bionic/netrc_${NETRC_VERSION}_amd64.deb/download.deb" -o "$ROOT_DIR/build/${NETRC_PACKAGE_NAME}"
+  fi
+
   PLUGN_VERSION=$(grep PLUGN_VERSION "${ROOT_DIR}/Makefile" | head -n1 | cut -d' ' -f3)
   PLUGN_PACKAGE_NAME="plugn_${PLUGN_VERSION}_amd64.deb"
   if [[ ! -f "$ROOT_DIR/build/${PLUGN_PACKAGE_NAME}" ]]; then
@@ -52,6 +58,7 @@ install_dependencies() {
   sudo dpkg -i \
     "${ROOT_DIR}/build/$DOCKER_IMAGE_LABELER_PACKAGE_NAME" \
     "${ROOT_DIR}/build/$HEROKUISH_PACKAGE_NAME" \
+    "${ROOT_DIR}/build/$NETRC_PACKAGE_NAME" \
     "${ROOT_DIR}/build/$PLUGN_PACKAGE_NAME" \
     "${ROOT_DIR}/build/$PROCFILE_UTIL_PACKAGE_NAME" \
     "${ROOT_DIR}/build/$SIGIL_PACKAGE_NAME" \

diff --git a/tests/unit/git_3.bats b/tests/unit/git_3.bats
@@ -5,12 +5,16 @@ load test_helper
 setup() {
   global_setup
   create_app
+  [[ -f "$DOKKU_ROOT/.netrc" ]] && cp -fp "$DOKKU_ROOT/.netrc" "$DOKKU_ROOT/.netrc.bak"
+  touch /home/dokku/.netrc
+  chown dokku:dokku /home/dokku/.netrc
   touch /home/dokku/.ssh/known_hosts
   chown dokku:dokku /home/dokku/.ssh/known_hosts
 }
 
 teardown() {
   rm -f /home/dokku/.ssh/id_rsa.pub || true
+  [[ -f "$DOKKU_ROOT/.netrc.bak" ]] && mv "$DOKKU_ROOT/.netrc.bak" "$DOKKU_ROOT/.netrc" && chown dokku:dokku "$DOKKU_ROOT/.netrc"
   destroy_app
   global_teardown
 }
@@ -60,6 +64,31 @@ teardown() {
   assert_equal "$output" "$((start_lines + 2))"
 }
 
+@test "(git) git:auth" {
+  run /bin/bash -c "dokku git:auth"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+
+  run /bin/bash -c "dokku git:auth github.com"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "Removing netrc auth entry for host github.com"
+
+  run /bin/bash -c "dokku git:auth github.com username"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+  assert_output_contains "Missing password for netrc auth entry"
+
+  run /bin/bash -c "dokku git:auth github.com username password"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+  assert_output_contains "Setting netrc auth entry for host github.com"
+}
+
 @test "(git) git:sync new [errors]" {
   run /bin/bash -c "dokku git:sync"
   echo "output: $output"
@@ -274,11 +303,40 @@ teardown() {
   assert_output_contains "Application deployed"
 }
 
+
+@test "(git) git:sync private" {
+  if [[ -z "$SYNC_GITHUB_USERNAME" ]] || [[ -z "$SYNC_GITHUB_PASSWORD" ]]; then
+    skip "skipping due to missing github credentials SYNC_GITHUB_USERNAME:SYNC_GITHUB_PASSWORD"
+  fi
+
+  run /bin/bash -c "dokku git:sync $TEST_APP https://github.com/dokku/smoke-test-app-private.git"
+  echo "output: $output"
+  echo "status: $status"
+  assert_failure
+  assert_output_contains "fatal: could not read Username for"
+
+  run /bin/bash -c "dokku git:auth github.com $SYNC_GITHUB_USERNAME $SYNC_GITHUB_PASSWORD"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "cat /home/dokku/.netrc"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+
+  run /bin/bash -c "dokku git:sync $TEST_APP https://github.com/dokku/smoke-test-app-private.git"
+  echo "output: $output"
+  echo "status: $status"
+  assert_success
+}
+
 @test "(git) git:public-key" {
   run /bin/bash -c "dokku git:public-key"
   echo "output: $output"
   echo "status: $status"
   assert_failure
+  assert_output_contains "There is no deploy key associated with the dokku user."
 
   run /bin/bash -c "cp /root/.ssh/dokku_test_rsa.pub /home/dokku/.ssh/id_rsa.pub"
   echo "output: $output"