diff --git a/docs/deployment/schedulers/k3s.md b/docs/deployment/schedulers/k3s.md index a63ca3e3a1d..a85c189a626 100644 --- a/docs/deployment/schedulers/k3s.md +++ b/docs/deployment/schedulers/k3s.md @@ -320,6 +320,38 @@ By default, Dokku assumes that all it controls all actions on the cluster, and t dokku scheduler-k3s:show-kubeconfig ``` +### Interacting with an external Kubernetes cluster + +While the k3s scheduler plugin is designed to work with a Dokku-managed k3s cluster, Dokku can be configured to interact with any Kubernetes cluster by setting the global `kubeconfig-path` to a path to a custom kubeconfig on the Dokku server. This property is only available at a global level. + +```shell +dokku scheduler-k3s:set --global kubeconfig-path /path/to/custom/kubeconfig +``` + +To set the default value, omit the value from the `scheduler-k3s:set` call: + +```shell +dokku scheduler-k3s:set --global kubeconfig-path +``` + +The default value for the `kubeconfig-path` is the k3s kubeconfig located at `/etc/rancher/k3s/k3s.yaml`. + +### Customizing the Kubernetes context + +When interacting with a custom Kubeconfig, the `kube-context` property can be set to specify a specific context within the kubeconfig to use. This property is available only at the global leve. + +```shell +dokku scheduler-k3s:set --global kube-context lollipop +``` + +To set the default value, omit the value from the `scheduler-k3s:set` call: + +```shell +dokku scheduler-k3s:set --global kube-context +``` + +The default value for the `kube-context` is an empty string, and will result in Dokku using the current context within the kubeconfig. + ## Scheduler Interface The following sections describe implemented and unimplemented scheduler functionality for the `k3s` scheduler. diff --git a/plugins/scheduler-k3s/functions.go b/plugins/scheduler-k3s/functions.go index 365d1268d88..02648d06826 100644 --- a/plugins/scheduler-k3s/functions.go +++ b/plugins/scheduler-k3s/functions.go @@ -1142,18 +1142,38 @@ func installHelm(ctx context.Context) error { return nil } +// isKubernetesAvailable returns an error if kubernetes api is not available +func isKubernetesAvailable() error { + client, err := NewKubernetesClient() + if err != nil { + return fmt.Errorf("Error creating kubernetes client: %w", err) + } + + if err := client.Ping(); err != nil { + return fmt.Errorf("Error pinging kubernetes: %w", err) + } + + return nil +} + +// isK3sInstalled returns an error if k3s is not installed func isK3sInstalled() error { if !common.FileExists("/usr/local/bin/k3s") { return fmt.Errorf("k3s binary is not available") } - if !common.FileExists(KubeConfigPath) { + if !common.FileExists(getKubeconfigPath()) { return fmt.Errorf("k3s kubeconfig is not available") } return nil } +// isK3sKubernetes returns true if the current kubernetes cluster is configured to be k3s +func isK3sKubernetes() bool { + return getKubeconfigPath() == KubeConfigPath +} + func isPodReady(ctx context.Context, clientset KubernetesClient, podName, namespace string) wait.ConditionWithContextFunc { return func(ctx context.Context) (bool, error) { fmt.Printf(".") diff --git a/plugins/scheduler-k3s/helm.go b/plugins/scheduler-k3s/helm.go index bf16fa1dce4..8752961057f 100644 --- a/plugins/scheduler-k3s/helm.go +++ b/plugins/scheduler-k3s/helm.go @@ -75,7 +75,9 @@ func NewHelmAgent(namespace string, logger action.DebugLog) (*HelmAgent, error) helmDriver = "secrets" } - kubeConfig := kube.GetConfig(KubeConfigPath, "", namespace) + kubeconfigPath := getKubeconfigPath() + kubeContext := getKubeContext() + kubeConfig := kube.GetConfig(kubeconfigPath, kubeContext, namespace) if err := actionConfig.Init(kubeConfig, namespace, helmDriver, logger); err != nil { return nil, err } diff --git a/plugins/scheduler-k3s/k8s.go b/plugins/scheduler-k3s/k8s.go index 99483f883c3..556d8d4abf0 100644 --- a/plugins/scheduler-k3s/k8s.go +++ b/plugins/scheduler-k3s/k8s.go @@ -22,11 +22,22 @@ import ( "k8s.io/utils/ptr" ) +func getKubeconfigPath() string { + return common.PropertyGetDefault("scheduler-k3s", "--global", "kubeconfig-path", KubeConfigPath) +} + +func getKubeContext() string { + return common.PropertyGetDefault("scheduler-k3s", "--global", "kube-context", DefaultKubeContext) +} + // KubernetesClient is a wrapper around the Kubernetes client type KubernetesClient struct { // Client is the Kubernetes client Client kubernetes.Clientset + // KubeConfigPath is the path to the Kubernetes config + KubeConfigPath string + // RestClient is the Kubernetes REST client RestClient rest.Interface @@ -36,7 +47,9 @@ type KubernetesClient struct { // NewKubernetesClient creates a new Kubernetes client func NewKubernetesClient() (KubernetesClient, error) { - clientConfig := KubernetesClientConfig() + kubeconfigPath := getKubeconfigPath() + kubeContext := getKubeContext() + clientConfig := KubernetesClientConfig(kubeconfigPath, kubeContext) restConf, err := clientConfig.ClientConfig() if err != nil { return KubernetesClient{}, err @@ -60,17 +73,29 @@ func NewKubernetesClient() (KubernetesClient, error) { } return KubernetesClient{ - Client: *client, - RestConfig: *restConf, - RestClient: restClient, + Client: *client, + KubeConfigPath: kubeconfigPath, + RestConfig: *restConf, + RestClient: restClient, }, nil } // KubernetesClientConfig returns a Kubernetes client config -func KubernetesClientConfig() clientcmd.ClientConfig { +func KubernetesClientConfig(kubeconfigPath string, kubecontext string) clientcmd.ClientConfig { + configOverrides := clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: ""}} + if kubecontext != "" { + configOverrides.CurrentContext = kubecontext + } + return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - &clientcmd.ClientConfigLoadingRules{ExplicitPath: KubeConfigPath}, - &clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: ""}}) + &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath}, + &configOverrides, + ) +} + +func (k KubernetesClient) Ping() error { + _, err := k.Client.Discovery().ServerVersion() + return err } // AnnotateNodeInput contains all the information needed to annotates a Kubernetes node @@ -110,16 +135,23 @@ type ApplyKubernetesManifestInput struct { } func (k KubernetesClient) ApplyKubernetesManifest(ctx context.Context, input ApplyKubernetesManifestInput) error { + args := []string{ + "apply", + "-f", + input.Manifest, + } + + if kubeContext := getKubeContext(); kubeContext != "" { + args = append([]string{"--context", kubeContext}, args...) + } + + if kubeconfigPath := getKubeconfigPath(); kubeconfigPath != "" { + args = append([]string{"--kubeconfig", kubeconfigPath}, args...) + } + upgradeCmd, err := common.CallExecCommand(common.ExecCommandInput{ - Command: "kubectl", - Args: []string{ - "apply", - "-f", - input.Manifest, - }, - Env: map[string]string{ - "KUBECONFIG": KubeConfigPath, - }, + Command: "kubectl", + Args: args, StreamStdio: true, }) if err != nil { diff --git a/plugins/scheduler-k3s/report.go b/plugins/scheduler-k3s/report.go index 25474883b88..2e25445dfa7 100644 --- a/plugins/scheduler-k3s/report.go +++ b/plugins/scheduler-k3s/report.go @@ -17,6 +17,8 @@ func ReportSingleApp(appName string, format string, infoFlag string) error { "--scheduler-k3s-computed-image-pull-secrets": reportComputedImagePullSecrets, "--scheduler-k3s-image-pull-secrets": reportImagePullSecrets, "--scheduler-k3s-global-image-pull-secrets": reportGlobalImagePullSecrets, + "--scheduler-k3s-global-kubeconfig-path": reportGlobalKubeconfigPath, + "--scheduler-k3s-global-kube-context": reportGlobalKubeContext, "--scheduler-k3s-computed-letsencrypt-server": reportComputedLetsencryptServer, "--scheduler-k3s-letsencrypt-server": reportLetsencryptServer, "--scheduler-k3s-global-letsencrypt-server": reportGlobalLetsencryptServer, @@ -71,6 +73,13 @@ func reportGlobalIngressClass(appName string) string { return getGlobalIngressClass() } +func reportGlobalKubeconfigPath(appName string) string { + return getKubeconfigPath() +} + +func reportGlobalKubeContext(appName string) string { + return getKubeContext() +} func reportComputedLetsencryptServer(appName string) string { return getComputedLetsencryptServer(appName) } diff --git a/plugins/scheduler-k3s/scheduler_k3s.go b/plugins/scheduler-k3s/scheduler_k3s.go index 0aca6c1b308..9064124fc66 100644 --- a/plugins/scheduler-k3s/scheduler_k3s.go +++ b/plugins/scheduler-k3s/scheduler_k3s.go @@ -29,6 +29,8 @@ var ( "deploy-timeout": true, "image-pull-secrets": true, "ingress-class": true, + "kube-context": true, + "kubeconfig-path": true, "letsencrypt-server": true, "letsencrypt-email-prod": true, "letsencrypt-email-stag": true, @@ -42,6 +44,7 @@ var ( const DefaultIngressClass = "traefik" const GlobalProcessType = "--global" const KubeConfigPath = "/etc/rancher/k3s/k3s.yaml" +const DefaultKubeContext = "" var ( runtimeScheme = runtime.NewScheme() diff --git a/plugins/scheduler-k3s/subcommands.go b/plugins/scheduler-k3s/subcommands.go index 594c938eb85..dd85668c7da 100644 --- a/plugins/scheduler-k3s/subcommands.go +++ b/plugins/scheduler-k3s/subcommands.go @@ -315,7 +315,16 @@ func CommandInitialize(ingressClass string, serverIP string, taintScheduling boo // CommandClusterAdd adds a server to the k3s cluster func CommandClusterAdd(role string, remoteHost string, serverIP string, allowUknownHosts bool, taintScheduling bool) error { if err := isK3sInstalled(); err != nil { - return fmt.Errorf("k3s not installed, cannot join cluster") + return fmt.Errorf("k3s not installed, cannot add node to cluster: %w", err) + } + + clientset, err := NewKubernetesClient() + if err != nil { + return fmt.Errorf("Unable to create kubernetes client: %w", err) + } + + if err := clientset.Ping(); err != nil { + return fmt.Errorf("kubernetes api not available, cannot add node to cluster: %w", err) } if role != "server" && role != "worker" { @@ -530,11 +539,6 @@ func CommandClusterAdd(role string, remoteHost string, serverIP string, allowUkn return fmt.Errorf("Invalid exit code from k3s installer command over ssh: %d", joinCmd.ExitCode) } - clientset, err := NewKubernetesClient() - if err != nil { - return fmt.Errorf("Unable to create kubernetes client: %w", err) - } - common.LogInfo2Quiet("Waiting for node to exist") nodes, err := waitForNodeToExist(ctx, WaitForNodeToExistInput{ Clientset: clientset, @@ -588,9 +592,6 @@ func CommandClusterList(format string) error { if format != "stdout" && format != "json" { return fmt.Errorf("Invalid format: %s", format) } - if err := isK3sInstalled(); err != nil { - return fmt.Errorf("k3s not installed, cannot list cluster nodes") - } ctx, cancel := context.WithCancel(context.Background()) signals := make(chan os.Signal, 1) @@ -608,6 +609,10 @@ func CommandClusterList(format string) error { return fmt.Errorf("Unable to create kubernetes client: %w", err) } + if err := clientset.Ping(); err != nil { + return fmt.Errorf("kubernetes api not available, cannot list cluster nodes: %w", err) + } + nodes, err := clientset.ListNodes(ctx, ListNodesInput{}) if err != nil { return fmt.Errorf("Unable to list nodes: %w", err) @@ -641,7 +646,7 @@ func CommandClusterList(format string) error { // CommandClusterRemove removes a node from the k3s cluster func CommandClusterRemove(nodeName string) error { if err := isK3sInstalled(); err != nil { - return fmt.Errorf("k3s not installed, cannot remove node") + return fmt.Errorf("k3s not installed, cannot remove node from cluster: %w", err) } ctx, cancel := context.WithCancel(context.Background()) @@ -661,6 +666,10 @@ func CommandClusterRemove(nodeName string) error { return fmt.Errorf("Unable to create kubernetes client: %w", err) } + if err := clientset.Ping(); err != nil { + return fmt.Errorf("kubernetes api not available: %w", err) + } + common.LogVerboseQuiet("Getting node remote connection information") node, err := clientset.GetNode(ctx, GetNodeInput{ Name: nodeName, @@ -730,11 +739,12 @@ func CommandSet(appName string, property string, value string) error { // CommandShowKubeconfig displays the kubeconfig file contents func CommandShowKubeconfig() error { - if !common.FileExists(KubeConfigPath) { - return fmt.Errorf("Kubeconfig file does not exist: %s", KubeConfigPath) + kubeconfigPath := getKubeconfigPath() + if !common.FileExists(kubeconfigPath) { + return fmt.Errorf("Kubeconfig file does not exist: %s", kubeconfigPath) } - b, err := os.ReadFile(KubeConfigPath) + b, err := os.ReadFile(kubeconfigPath) if err != nil { return fmt.Errorf("Unable to read kubeconfig file: %w", err) } @@ -746,7 +756,7 @@ func CommandShowKubeconfig() error { func CommandUninstall() error { if err := isK3sInstalled(); err != nil { - return fmt.Errorf("k3s not installed, cannot uninstall") + return fmt.Errorf("k3s not installed, cannot uninstall: %w", err) } common.LogInfo1("Uninstalling k3s") diff --git a/plugins/scheduler-k3s/triggers.go b/plugins/scheduler-k3s/triggers.go index f0578468a97..073b13eb10e 100644 --- a/plugins/scheduler-k3s/triggers.go +++ b/plugins/scheduler-k3s/triggers.go @@ -414,6 +414,10 @@ func TriggerSchedulerDeploy(scheduler string, appName string, imageTag string) e return fmt.Errorf("Error creating kubernetes client: %w", err) } + if err := clientset.Ping(); err != nil { + return fmt.Errorf("kubernetes api not available: %w", err) + } + cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{ LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName), Namespace: namespace, @@ -573,6 +577,10 @@ func TriggerSchedulerEnter(scheduler string, appName string, processType string, return fmt.Errorf("Error creating kubernetes client: %w", err) } + if err := clientset.Ping(); err != nil { + return fmt.Errorf("kubernetes api not available: %w", err) + } + namespace := getComputedNamespace(appName) labelSelector := []string{fmt.Sprintf("app.kubernetes.io/part-of=%s", appName)} processIndex := 1 @@ -664,6 +672,10 @@ func TriggerSchedulerLogs(scheduler string, appName string, processType string, return fmt.Errorf("Error creating kubernetes client: %w", err) } + if err := clientset.Ping(); err != nil { + return fmt.Errorf("kubernetes api not available: %w", err) + } + labelSelector := []string{fmt.Sprintf("app.kubernetes.io/part-of=%s", appName)} processIndex := 0 if processType != "" { @@ -926,6 +938,10 @@ func TriggerSchedulerRun(scheduler string, appName string, envCount int, args [] return fmt.Errorf("Error creating kubernetes client: %w", err) } + if err := clientset.Ping(); err != nil { + return fmt.Errorf("kubernetes api not available: %w", err) + } + ctx, cancel := context.WithCancel(context.Background()) signals := make(chan os.Signal, 1) signal.Notify(signals, os.Interrupt, syscall.SIGHUP, @@ -1078,6 +1094,10 @@ func TriggerSchedulerRunList(scheduler string, appName string, format string) er return fmt.Errorf("Error creating kubernetes client: %w", err) } + if err := clientset.Ping(); err != nil { + return fmt.Errorf("kubernetes api not available: %w", err) + } + namespace := getComputedNamespace(appName) cronJobs, err := clientset.ListCronJobs(ctx, ListCronJobsInput{ LabelSelector: fmt.Sprintf("app.kubernetes.io/part-of=%s", appName), @@ -1139,9 +1159,26 @@ func TriggerSchedulerPostDelete(scheduler string, appName string) error { return nil } - if err := isK3sInstalled(); err != nil { - common.LogWarn(fmt.Sprintf("Skipping app deletion: %s", err.Error())) - return nil + dataErr := common.RemoveAppDataDirectory("logs", appName) + propertyErr := common.PropertyDestroy("logs", appName) + + if dataErr != nil { + return dataErr + } + + if propertyErr != nil { + return propertyErr + } + + if isK3sKubernetes() { + if err := isK3sInstalled(); err != nil { + common.LogWarn("k3s is not installed, skipping") + return nil + } + } + + if err := isKubernetesAvailable(); err != nil { + return fmt.Errorf("kubernetes api not available: %w", err) } namespace := getComputedNamespace(appName) @@ -1164,11 +1201,6 @@ func TriggerSchedulerStop(scheduler string, appName string) error { return nil } - if err := isK3sInstalled(); err != nil { - common.LogWarn(fmt.Sprintf("Skipping app stop: %s", err.Error())) - return nil - } - ctx, cancel := context.WithCancel(context.Background()) signals := make(chan os.Signal, 1) signal.Notify(signals, os.Interrupt, syscall.SIGHUP, @@ -1182,9 +1214,19 @@ func TriggerSchedulerStop(scheduler string, appName string) error { clientset, err := NewKubernetesClient() if err != nil { + if isK3sKubernetes() { + if err := isK3sInstalled(); err != nil { + common.LogWarn("k3s is not installed, skipping") + return nil + } + } return fmt.Errorf("Error creating kubernetes client: %w", err) } + if err := clientset.Ping(); err != nil { + return fmt.Errorf("kubernetes api not available: %w", err) + } + namespace := getComputedNamespace(appName) deployments, err := clientset.ListDeployments(ctx, ListDeploymentsInput{ Namespace: namespace,