diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index 1f874c8c108..2846949e999 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -568,6 +568,11 @@ This plugin implements various functionality through `plugn` triggers to integra - `apps:clone` - `apps:destroy` - `apps:rename` +- `docker-options`: + - The following docker options are translated into their kubernetes equivalents: + - `--cap-add` + - `--cap-drop` + - `--privileged` - `cron` - `enter` - `deploy` diff --git a/plugins/docker-options/dockeroptions.go b/plugins/docker-options/dockeroptions.go index e9738637769..00c0fc90647 100644 --- a/plugins/docker-options/dockeroptions.go +++ b/plugins/docker-options/dockeroptions.go @@ -6,6 +6,8 @@ import ( "os" "sort" "strings" + + "github.com/dokku/dokku/plugins/common" ) // SetDockerOptionForPhases sets an option to specified phases @@ -85,3 +87,57 @@ func GetDockerOptionsForPhase(appName string, phase string) ([]string, error) { return options, nil } + +// GetSpecifiedDockerOptionsForPhase returns the docker options for the specified phase that are in the desiredOptions list +// It expects desiredOptions to be a list of docker options that are in the format "--option" +// And will retrieve any lines that start with the desired option +func GetSpecifiedDockerOptionsForPhase(appName string, phase string, desiredOptions []string) (map[string][]string, error) { + foundOptions := map[string][]string{} + options, err := GetDockerOptionsForPhase(appName, phase) + if err != nil { + return foundOptions, err + } + + for _, option := range options { + for _, desiredOption := range desiredOptions { + if option == desiredOption { + foundOptions[desiredOption] = []string{} + break + } + + // match options that are in the format "--option=value" + if strings.HasPrefix(option, fmt.Sprintf("%s=", desiredOption)) { + if _, ok := foundOptions[desiredOption]; !ok { + foundOptions[desiredOption] = []string{} + } + + parts := strings.SplitN(option, "=", 2) + if len(parts) != 2 { + common.LogWarn(fmt.Sprintf("Invalid docker option found for %s: %s", appName, option)) + continue + } + + foundOptions[desiredOption] = append(foundOptions[desiredOption], parts[1]) + break + } + + // match options that are in the format "--option value" + if strings.HasPrefix(option, fmt.Sprintf("%s ", desiredOption)) { + if _, ok := foundOptions[desiredOption]; !ok { + foundOptions[desiredOption] = []string{} + } + + parts := strings.SplitN(option, " ", 2) + if len(parts) != 2 { + common.LogWarn(fmt.Sprintf("Invalid docker option found for %s: %s", appName, option)) + continue + } + + foundOptions[desiredOption] = append(foundOptions[desiredOption], parts[1]) + break + } + } + } + + return foundOptions, nil +} diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index e163ee0022f..fe718153bc0 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -18,6 +18,7 @@ import ( appjson "github.com/dokku/dokku/plugins/app-json" "github.com/dokku/dokku/plugins/common" + dockeroptions "github.com/dokku/dokku/plugins/docker-options" "github.com/dokku/dokku/plugins/logs" nginxvhosts "github.com/dokku/dokku/plugins/nginx-vhosts" resty "github.com/go-resty/resty/v2" @@ -1489,6 +1490,37 @@ func getStartCommand(input StartCommandInput) (StartCommandOutput, error) { }, nil } +func getSecurityContext(appName string, phase string) (SecurityContext, error) { + securityContext := SecurityContext{} + deployOptions, err := dockeroptions.GetSpecifiedDockerOptionsForPhase(appName, phase, []string{ + "--cap-add", + "--cap-drop", + "--privileged", + }) + if err != nil { + return SecurityContext{}, fmt.Errorf("Error getting deploy options: %w", err) + } + + if _, ok := deployOptions["--privileged"]; ok { + securityContext.Privileged = true + } + if capAdd, ok := deployOptions["--cap-add"]; ok { + capabilities := []string{} + for _, cap := range capAdd { + capabilities = append(capabilities, strings.ToUpper(cap)) + } + securityContext.Capabilities.Add = capabilities + } + if capDrop, ok := deployOptions["--cap-drop"]; ok { + capabilities := []string{} + for _, cap := range capDrop { + capabilities = append(capabilities, strings.ToUpper(cap)) + } + securityContext.Capabilities.Drop = capabilities + } + return securityContext, nil +} + func getProcessSpecificKustomizeRootPath(appName string) string { if !hasKustomizeDirectory(appName) { return "" diff --git a/plugins/scheduler-k3s/template.go b/plugins/scheduler-k3s/template.go index 8ce06080665..835e5f35df8 100644 --- a/plugins/scheduler-k3s/template.go +++ b/plugins/scheduler-k3s/template.go @@ -3,6 +3,7 @@ package scheduler_k3s import ( "crypto/rand" "fmt" + "maps" "os" "strings" @@ -41,15 +42,16 @@ type AppValues struct { } type GlobalValues struct { - Annotations ProcessAnnotations `yaml:"annotations,omitempty"` - AppName string `yaml:"app_name"` - DeploymentID string `yaml:"deployment_id"` - Image GlobalImage `yaml:"image"` - Labels ProcessLabels `yaml:"labels,omitempty"` - Keda GlobalKedaValues `yaml:"keda"` - Namespace string `yaml:"namespace"` - Network GlobalNetwork `yaml:"network"` - Secrets map[string]string `yaml:"secrets,omitempty"` + Annotations ProcessAnnotations `yaml:"annotations,omitempty"` + AppName string `yaml:"app_name"` + DeploymentID string `yaml:"deployment_id"` + Image GlobalImage `yaml:"image"` + Labels ProcessLabels `yaml:"labels,omitempty"` + Keda GlobalKedaValues `yaml:"keda"` + Namespace string `yaml:"namespace"` + Network GlobalNetwork `yaml:"network"` + Secrets map[string]string `yaml:"secrets,omitempty"` + SecurityContext SecurityContext `yaml:"security_context,omitempty"` } type GlobalImage struct { @@ -351,11 +353,53 @@ type Job struct { Namespace string ProcessType string Schedule string + SecurityContext SecurityContext Suffix string RemoveContainer bool WorkingDir string } +// SecurityContext contains the security context for a process +type SecurityContext struct { + // Capabilities contains the capabilities for a process + Capabilities SecurityContextCapabilities `yaml:"capabilities,omitempty"` + // Privileged contains the privileged flag for a process + Privileged bool `yaml:"privileged,omitempty"` +} + +// ToCoreV1SecurityContext converts the security context to a corev1.SecurityContext +func (s SecurityContext) ToCoreV1SecurityContext() corev1.SecurityContext { + securityContext := corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{}, + Privileged: ptr.To(s.Privileged), + } + + if len(s.Capabilities.Add) > 0 { + capabilities := make([]corev1.Capability, len(s.Capabilities.Add)) + for i, cap := range s.Capabilities.Add { + capabilities[i] = corev1.Capability(cap) + } + securityContext.Capabilities.Add = capabilities + } + if len(s.Capabilities.Drop) > 0 { + capabilities := make([]corev1.Capability, len(s.Capabilities.Drop)) + for i, cap := range s.Capabilities.Drop { + capabilities[i] = corev1.Capability(cap) + } + securityContext.Capabilities.Drop = capabilities + } + + return securityContext +} + +// SecurityContextCapabilities contains the capabilities for a process +type SecurityContextCapabilities struct { + // Add contains the add capabilities for a process + Add []string `yaml:"add,omitempty"` + // Drop contains the drop capabilities for a process + Drop []string `yaml:"drop,omitempty"` +} + func templateKubernetesJob(input Job) (batchv1.Job, error) { labels := map[string]string{ "app.kubernetes.io/instance": fmt.Sprintf("%s-%s", input.AppName, input.ProcessType), @@ -368,9 +412,7 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) { "dokku.com/managed": "true", } - for key, value := range input.Labels { - labels[key] = value - } + maps.Copy(labels, input.Labels) secretName := fmt.Sprintf("env-%s.%d", input.AppName, input.DeploymentID) env := []corev1.EnvVar{} @@ -417,6 +459,8 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) { podAnnotations[key] = value } + securityContext := input.SecurityContext.ToCoreV1SecurityContext() + job := batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s-%s", input.AppName, input.ProcessType, suffix), @@ -453,7 +497,8 @@ func templateKubernetesJob(input Job) (batchv1.Job, error) { Limits: corev1.ResourceList{}, Requests: corev1.ResourceList{}, }, - WorkingDir: input.WorkingDir, + SecurityContext: &securityContext, + WorkingDir: input.WorkingDir, }, }, RestartPolicy: corev1.RestartPolicyNever, diff --git a/plugins/scheduler-k3s/templates/chart/cron-job.yaml b/plugins/scheduler-k3s/templates/chart/cron-job.yaml index 7aedc8be2c5..f44d4069659 100644 --- a/plugins/scheduler-k3s/templates/chart/cron-job.yaml +++ b/plugins/scheduler-k3s/templates/chart/cron-job.yaml @@ -102,6 +102,27 @@ spec: {{- end }} {{- end }} {{- end }} + {{- if hasKey $.Values.global "security_context" }} + securityContext: + {{- if $.Values.global.security_context.privileged }} + privileged: true + {{- end }} + {{- if hasKey $.Values.global.security_context "capabilities" }} + capabilities: + {{- if hasKey $.Values.global.security_context.capabilities "add" }} + add: + {{- range $.Values.global.security_context.capabilities.add }} + - {{ . }} + {{- end }} + {{- end }} + {{- if hasKey $.Values.global.security_context.capabilities "drop" }} + drop: + {{- range $.Values.global.security_context.capabilities.drop }} + - {{ . }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} {{- if $.Values.global.image.working_dir }} workingDir: {{ $.Values.global.image.working_dir }} {{- end }} diff --git a/plugins/scheduler-k3s/templates/chart/deployment.yaml b/plugins/scheduler-k3s/templates/chart/deployment.yaml index 30a38ec1214..10455a32979 100644 --- a/plugins/scheduler-k3s/templates/chart/deployment.yaml +++ b/plugins/scheduler-k3s/templates/chart/deployment.yaml @@ -117,6 +117,27 @@ spec: readinessProbe: {{ $config.healthchecks.readiness | toJson | indent 10 }} {{- end }} + {{- if hasKey $.Values.global "security_context" }} + securityContext: + {{- if $.Values.global.security_context.privileged }} + privileged: true + {{- end }} + {{- if hasKey $.Values.global.security_context "capabilities" }} + capabilities: + {{- if hasKey $.Values.global.security_context.capabilities "add" }} + add: + {{- range $.Values.global.security_context.capabilities.add }} + - {{ . }} + {{- end }} + {{- end }} + {{- if hasKey $.Values.global.security_context.capabilities "drop" }} + drop: + {{- range $.Values.global.security_context.capabilities.drop }} + - {{ . }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} {{- if $.Values.global.image.working_dir }} workingDir: {{ $.Values.global.image.working_dir }} {{- end }} diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index 0207d42f3e6..298ee7fe26a 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -387,6 +387,11 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e return fmt.Errorf("Error getting keda values: %w", err) } + securityContext, err := getSecurityContext(appName, "deploy") + if err != nil { + return fmt.Errorf("Error getting security context: %w", err) + } + values := &AppValues{ Global: GlobalValues{ Annotations: globalAnnotations, @@ -407,7 +412,8 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e PrimaryPort: primaryPort, PrimaryServicePort: primaryServicePort, }, - Secrets: map[string]string{}, + Secrets: map[string]string{}, + SecurityContext: securityContext, }, Processes: map[string]ProcessValues{}, } @@ -1228,6 +1234,11 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args [] } } + securityContext, err := getSecurityContext(appName, "run") + if err != nil { + return fmt.Errorf("Error getting security context: %w", err) + } + workingDir := common.GetWorkingDir(appName, image) job, err := templateKubernetesJob(Job{ AppName: appName, @@ -1243,6 +1254,7 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args [] Namespace: namespace, ProcessType: processType, RemoveContainer: rmContainer, + SecurityContext: securityContext, WorkingDir: workingDir, }) if err != nil { diff --git a/tests/unit/scheduler-k3s-3.bats b/tests/unit/scheduler-k3s-3.bats index 4a6a3fb074d..1f6be842800 100644 --- a/tests/unit/scheduler-k3s-3.bats +++ b/tests/unit/scheduler-k3s-3.bats @@ -11,12 +11,69 @@ setup() { export KUBECONFIG="/etc/rancher/k3s/k3s.yaml" } -teardown_() { +teardown() { global_teardown dokku nginx:start uninstall_k3s || true } +@test "(scheduler-k3s) security context" { + if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then + skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN" + fi + + INGRESS_CLASS=nginx install_k3s + + run /bin/bash -c "dokku apps:create $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run deploy_app python "dokku@$DOKKU_DOMAIN:$TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.spec.containers[0].securityContext.capabilities.add[0]'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "null" + + run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.spec.containers[0].securityContext.privileged'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "null" + + run /bin/bash -c "dokku docker-options:add $TEST_APP deploy --cap-add=NET_ADMIN" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku docker-options:add $TEST_APP deploy --privileged" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "dokku ps:restart $TEST_APP" + echo "output: $output" + echo "status: $status" + assert_success + + run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.spec.containers[0].securityContext.capabilities.add[0]'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "NET_ADMIN" + + run /bin/bash -c "kubectl get deployment $TEST_APP-web -o json | jq -r '.spec.template.spec.containers[0].securityContext.privileged'" + echo "output: $output" + echo "status: $status" + assert_success + assert_output "true" +} + @test "(scheduler-k3s) kustomize" { if [[ -z "$DOCKERHUB_USERNAME" ]] || [[ -z "$DOCKERHUB_TOKEN" ]]; then skip "skipping due to missing docker.io credentials DOCKERHUB_USERNAME:DOCKERHUB_TOKEN"