diff --git a/README.md b/README.md index cda697a..a04e35d 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This tool provide simple auto horizontal scaling on Upsun project for your appli > [!CAUTION] > **This project is owned by the Upsun Advocacy team. It is in early stage of development [experimental] and only intended to be used with caution by Upsun customers/community.

This project is not supported by Upsun and does not qualify for Support plans. Use this repository at your own risks, it is provided without guarantee or warranty!** > -> PS: if you have downloaded that binary and appreciated it, I kindly ask you to let us know on [Discord](https://discord.com/invite/platformsh) (you can then later discuss with one of our Product manager to share your feedback on it). +> PS: if you have downloaded that binary and appreciated it, I kindly ask you to let us know on [Discord](https://discord.gg/upsun) (you can then later discuss with one of our Product manager to share your feedback on it). ## Usage/install @@ -44,6 +44,10 @@ env:UPSUN_CLI_TOKEN ``` Usage of scalsun: --name string Apps or Service name + --include_service Autoscale the services + --type string Type of scaling (horizontal or vertical or timming) (default "horizontal") + --min_size_count: float Minimum host size (default 0.1) + --max_size_count float Maximum host size (default 8) --min_host_count: int Minimum host count (default 1) --max_host_count int Maximum host count (default 3) --min_cpu_usage_upscale float Minimum CPU usage in % (for upscale event only) (default 75) @@ -56,7 +60,14 @@ Usage of scalsun: ``` #### Samples -- Auto-scale all app/service +- Auto-scale (Horizontal) all app/service `scalsun --silent --max_host_count=${H_SCALING_HOST_MAX:-3}` -- Auto-scale only specific app (if app name is web) -`scalsun --silent --max_host_count=${H_SCALING_HOST_MAX:-3} --name=web` + +- Auto-scale (Horizontal) only specific app (if app name is web) +`scalsun --silent --type=horizontal --max_host_count=${H_SCALING_HOST_MAX:-3} --name=web` + +- Auto-scale (Vertical) only specific app (if app name is web) +`scalsun --silent --type=vertical --max_cpu_usage_downscale=20 --max_host_size=1 --min_host_size=0.1 --name=web` + +- Auto-scale (Custom) only specific app (if app name is web) +`scalsun --silent --type=custom --max_host_size=1 --max_host_count=0.1 --name=web` diff --git a/cmd/scaling-instance.go b/cmd/scaling-instance.go index eafb074..53d32c3 100644 --- a/cmd/scaling-instance.go +++ b/cmd/scaling-instance.go @@ -1,8 +1,10 @@ package main import ( + "fmt" "log" "os" + "strings" flag "github.com/spf13/pflag" lib "github.com/upsun/lib-sun" @@ -13,14 +15,29 @@ import ( ) const ( - APP_NAME = "scalsun" + APP_NAME = "scalsun" + TYPE_HORIZONTAL = "horizontal" + TYPE_VERTICAL = "vertical" + TYPE_CUSTOM = "custom" ) func init() { + // Common arguments flag.StringVarP(&app.ArgsS.Name, "name", "", "", "Apps or Service name") - flag.BoolVarP(&app.ArgsS.IncludeServices, "include_service", "", false, "Autoscale the services") - flag.IntVarP(&app.ArgsS.HostCountMin, "min_host_count:", "", 1, "Minimum host count") + flag.BoolVarP(&app.ArgsS.IncludeServices, "include_service", "", false, "Autoscale the services (not applicable)") + + flag.StringVarP(&app.ArgsS.Type, "type", "", TYPE_HORIZONTAL, fmt.Sprintf("Type of scaling (%s or %s or %s)", TYPE_HORIZONTAL, TYPE_VERTICAL, TYPE_CUSTOM)) + + // Vertical scaling arguments + flag.Float64VarP(&app.ArgsS.HostSizeMin, "min_size_count", "", 0.1, "Minimum host size") + flag.Float64VarP(&app.ArgsS.HostSizeMax, "max_size_count", "", 8, "Maximum host size") + + // Horizontal scaling arguments + flag.IntVarP(&app.ArgsS.HostCountMin, "min_host_count", "", 1, "Minimum host count") flag.IntVarP(&app.ArgsS.HostCountMax, "max_host_count", "", 3, "Maximum host count") + + // Trigger + //flag.StringVarP(&app.ArgsS.RangeTime, "", "", "", "") flag.Float64VarP(&app.ArgsS.CpuUsageMin, "min_cpu_usage_upscale", "", 75.0, "Minimum CPU usage in % (for upscale event only)") flag.Float64VarP(&app.ArgsS.CpuUsageMax, "max_cpu_usage_downscale", "", 60.0, "Maximum CPU usage in % (for downscale event only)") flag.Float64VarP(&app.ArgsS.MemUsageMin, "min_mem_usage_upscale", "", 80.0, "Minimum memory usage in % (for upscale event only)") @@ -54,11 +71,25 @@ func main() { lib.Args = app.Args } + if !entity.IsValidType(app.ArgsS.Type) { + fmt.Fprintf(os.Stderr, "Invalid value for --type: %q\n", app.ArgsS.Type) + fmt.Fprintf(os.Stderr, "Valid values are: %s\n", strings.Join(entity.ValidTypes, ", ")) + os.Exit(1) + } + // Init projectContext := entity.MakeProjectContext( "upsun", projectID, branch, ) - logic.ScalingInstance(projectContext) + + switch app.ArgsS.Type { + case TYPE_CUSTOM: + logic.ForceContainerInstance(projectContext) + case TYPE_VERTICAL: + logic.ScalingContainer(projectContext) + case TYPE_HORIZONTAL: + logic.ScalingInstance(projectContext) + } } diff --git a/go.mod b/go.mod index 9e05313..42b3df0 100644 --- a/go.mod +++ b/go.mod @@ -13,5 +13,3 @@ require ( golang.org/x/text v0.14.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) - -//replace github.com/upsun/lib-sun => ../lib-sun diff --git a/internal/logic/scaling_resource.go b/internal/logic/scaling_resource.go index 0a0623b..f627dff 100644 --- a/internal/logic/scaling_resource.go +++ b/internal/logic/scaling_resource.go @@ -18,18 +18,277 @@ type UsageValue struct { } type UsageApp struct { - Name string - Values []UsageValue - Update bool - InstanceOld int - InstanceNew int + Name string + Values []UsageValue + + // Horizontal scaling + InstanceUpdate bool + InstanceOld int + InstanceNew int + + // Vertical scaling + IdProfileUpdate bool + IdProfileOld float64 + IdProfileNew SizeProfile +} + +type SizeProfile struct { + Id string + Cpu float64 + Mem int32 +} + +func ForceContainerInstance(projectContext entity.ProjectGlobal) { + usages := map[string]UsageApp{} + + log.Println("Process Force Scaling... (only use max_host_count & max_size_count argument)") + + GetContainersUsage(projectContext, usages) + RemoveUnscaleContainers(projectContext, usages) + hostSizeMax := app.ArgsS.HostSizeMax + hostCountMax := app.ArgsS.HostCountMax + + for _, usage := range usages { + profils := ListContainerSize(projectContext, usage.Name) + idProfilProposal := findBestUpProfile(profils, hostSizeMax, hostSizeMax) + + if usage.IdProfileOld != hostSizeMax { + usage.IdProfileNew = idProfilProposal + usage.IdProfileUpdate = true + } + + if usage.InstanceOld != hostCountMax { + usage.InstanceNew = hostCountMax + usage.InstanceUpdate = true + } + + usages[usage.Name] = usage + } + + SetResources(projectContext, usages) +} + +func ScalingContainer(projectContext entity.ProjectGlobal) { + usages := map[string]UsageApp{} + + log.Println("Process Vertical Auto-Scaling... (with down-time)") + + GetProjectMetrics(projectContext, usages) + GetContainersUsage(projectContext, usages) + RemoveUnscaleContainers(projectContext, usages) + + // Compute + log.Println("Compute Horizontal trend...") + for _, usage := range usages { + + //// Algo Threshold-limit + lastMetric := usage.Values[len(usage.Values)-1] + cpuUsageMin := app.ArgsS.CpuUsageMin + cpuUsageMax := app.ArgsS.CpuUsageMax + memUsageMin := app.ArgsS.MemUsageMin + memUsageMax := app.ArgsS.MemUsageMax + hostSizeMax := app.ArgsS.HostSizeMax + hostSizeMin := app.ArgsS.HostSizeMin + + profils := ListContainerSize(projectContext, usage.Name) + + // CPU case + if lastMetric.Cpu > cpuUsageMin && usage.IdProfileOld <= hostSizeMax { + idProfilCpuTheorical := usage.IdProfileOld * (lastMetric.Cpu / cpuUsageMin) + idProfilProposal := findBestUpProfile(profils, idProfilCpuTheorical, hostSizeMax) + if usage.IdProfileNew.Cpu < idProfilProposal.Cpu { + usage.IdProfileNew = idProfilProposal + if usage.IdProfileNew.Cpu != usage.IdProfileOld { + log.Printf("Upscale profile %v from %v to %v ! (Cpu: %v > %v)", usage.Name, usage.IdProfileOld, usage.IdProfileNew.Id, lastMetric.Cpu, cpuUsageMin) + usage.IdProfileUpdate = true + } + } + usages[usage.Name] = usage + } else if lastMetric.Cpu < cpuUsageMax && usage.IdProfileOld >= hostSizeMin { + idProfilCpuTheorical := usage.IdProfileOld * (lastMetric.Cpu / cpuUsageMax) + idProfilProposal := findBestDownProfile(profils, idProfilCpuTheorical, hostSizeMin) + if usage.IdProfileNew.Cpu < idProfilProposal.Cpu { + usage.IdProfileNew = idProfilProposal + if usage.IdProfileNew.Cpu != usage.IdProfileOld { + log.Printf("Downscale profile %v from %v to %v ! (Cpu: %v < %v)", usage.Name, usage.IdProfileOld, usage.IdProfileNew.Id, lastMetric.Cpu, cpuUsageMax) + usage.IdProfileUpdate = true + } + } + usages[usage.Name] = usage + } + + // Mem case + if lastMetric.Mem > memUsageMin && usage.IdProfileOld <= hostSizeMax { + idProfilCpuTheorical := usage.IdProfileOld * (lastMetric.Mem / memUsageMin) + idProfilProposal := findBestUpProfile(profils, idProfilCpuTheorical, hostSizeMax) + if usage.IdProfileNew.Cpu < idProfilProposal.Cpu { + usage.IdProfileNew = idProfilProposal + if usage.IdProfileNew.Cpu != usage.IdProfileOld { + log.Printf("Upscale profile %v from %v to %v ! (Mem: %v > %v)", usage.Name, usage.IdProfileOld, usage.IdProfileNew.Id, lastMetric.Mem, memUsageMin) + usage.IdProfileUpdate = true + } + } + usages[usage.Name] = usage + } else if lastMetric.Mem < memUsageMax && usage.IdProfileOld >= hostSizeMin { + idProfilCpuTheorical := usage.IdProfileOld * (lastMetric.Mem / memUsageMax) + idProfilProposal := findBestDownProfile(profils, idProfilCpuTheorical, hostSizeMin) + if usage.IdProfileNew.Cpu < idProfilProposal.Cpu { + usage.IdProfileNew = idProfilProposal + if usage.IdProfileNew.Cpu != usage.IdProfileOld { + log.Printf("Downscale profile %v from %v to %v ! (Mem: %v < %v)", usage.Name, usage.IdProfileOld, usage.IdProfileNew.Id, lastMetric.Mem, memUsageMax) + usage.IdProfileUpdate = true + } + } + usages[usage.Name] = usage + } + } + + SetResources(projectContext, usages) +} + +func findBestUpProfile(profils []SizeProfile, idProfilTheorical float64, hostSizeMax float64) SizeProfile { + profilProposal := profils[0] + + for idx, profile := range profils { + + if profile.Cpu > hostSizeMax { + profilProposal = profils[idx-1] + break + } + + if profile.Cpu >= idProfilTheorical { + profilProposal = profile + break + } + } + return profilProposal +} + +func findBestDownProfile(profils []SizeProfile, idProfilTheorical float64, hostSizeMin float64) SizeProfile { + profilProposal := profils[0] + + // Iterate in reverse order to find the first profile below the theoretical value + for idx := len(profils) - 1; idx >= 0; idx-- { + profile := profils[idx] + + if profile.Cpu < hostSizeMin { + profilProposal = profils[idx+1] + break + } + + if profile.Cpu <= idProfilTheorical { + profilProposal = profile + break + } + } + return profilProposal } func ScalingInstance(projectContext entity.ProjectGlobal) { usages := map[string]UsageApp{} - // Get metrics + log.Println("Process Horizontal Auto-Scaling... (without down-time)") + + GetProjectMetrics(projectContext, usages) + GetContainersUsage(projectContext, usages) + RemoveUnscaleContainers(projectContext, usages) + + // Compute + log.Println("Compute Horizontal trend...") + for _, usage := range usages { + + //// Algo Threshold-limit + lastMetric := usage.Values[len(usage.Values)-1] + + // CPU case + if lastMetric.Cpu > app.ArgsS.CpuUsageMin && usage.InstanceOld < app.ArgsS.HostCountMax { + instanceProposal := int(math.Ceil(float64(usage.InstanceOld) * (lastMetric.Cpu / app.ArgsS.CpuUsageMin))) + if usage.InstanceNew < instanceProposal { + usage.InstanceNew = instanceProposal + if usage.InstanceNew != usage.InstanceOld { + log.Printf("Upscale instance %v from %v to %v ! (Cpu: %v > %v)", usage.Name, usage.InstanceOld, usage.InstanceNew, lastMetric.Cpu, app.ArgsS.CpuUsageMin) + usage.InstanceUpdate = true + } + } + usages[usage.Name] = usage + } else if lastMetric.Cpu < app.ArgsS.CpuUsageMax && usage.InstanceOld > app.ArgsS.HostCountMin { + instanceProposal := int(math.Ceil(float64(usage.InstanceOld) * (lastMetric.Cpu / app.ArgsS.CpuUsageMax))) + if usage.InstanceNew < instanceProposal { + usage.InstanceNew = instanceProposal + if usage.InstanceNew != usage.InstanceOld { + log.Printf("Downscale instance %v from %v to %v ! (Cpu: %v < %v)", usage.Name, usage.InstanceOld, usage.InstanceNew, lastMetric.Cpu, app.ArgsS.CpuUsageMax) + usage.InstanceUpdate = true + } + } + usages[usage.Name] = usage + } + + // Mem case + if lastMetric.Mem > app.ArgsS.MemUsageMin && usage.InstanceOld < app.ArgsS.HostCountMax { + instanceProposal := int(math.Ceil(float64(usage.InstanceOld) * (lastMetric.Mem / app.ArgsS.MemUsageMin))) + if usage.InstanceNew < instanceProposal { + usage.InstanceNew = instanceProposal + if usage.InstanceNew != usage.InstanceOld { + log.Printf("Upscale instance %v from %v to %v ! (Mem: %v > %v)", usage.Name, usage.InstanceOld, usage.InstanceNew, lastMetric.Mem, app.ArgsS.MemUsageMin) + usage.InstanceUpdate = true + } + } + usages[usage.Name] = usage + } else if lastMetric.Mem < app.ArgsS.MemUsageMax && usage.InstanceOld > app.ArgsS.HostCountMin { + instanceProposal := int(math.Ceil(float64(usage.InstanceOld) * (lastMetric.Mem / app.ArgsS.MemUsageMax))) + if usage.InstanceNew < instanceProposal { + usage.InstanceNew = instanceProposal + if usage.InstanceNew != usage.InstanceOld { + log.Printf("Downscale instance %v from %v to %v ! (Mem: %v > %v)", usage.Name, usage.InstanceOld, usage.InstanceNew, lastMetric.Mem, app.ArgsS.MemUsageMax) + usage.InstanceUpdate = true + } + } + usages[usage.Name] = usage + } + } + + SetResources(projectContext, usages) +} + +// ListContainerSize retrieves the size profiles of containers in the project. +func ListContainerSize(projectContext entity.ProjectGlobal, name string) []SizeProfile { + log.Println("Get Size of Profile available...") + payload := []string{ + "--environment=" + projectContext.DefaultEnv, + "--service=" + name, + "--no-header", + "--format=csv", + "--no-interaction", + "--yes", + } + output, err := utils.CallCLIString(projectContext, "resources:size:list", payload...) + if err != nil { + log.Printf("No profile for this container") + } + + // Parse output + size_tpl := strings.Split(output, "\n") + var sizeProfiles []SizeProfile + for _, size_str := range size_tpl[:len(size_tpl)-1] { + size := strings.Split(size_str, ",") + + // Normalize + cpu, _ := strconv.ParseFloat(size[1], 64) + mem, _ := strconv.ParseInt(size[2], 10, 32) + profile := SizeProfile{ + Id: size[0], + Cpu: cpu, + Mem: int32(mem), + } + sizeProfiles = append(sizeProfiles, profile) + } + + return sizeProfiles +} + +func GetProjectMetrics(projectContext entity.ProjectGlobal, usages map[string]UsageApp) { log.Println("Get metrics of project...") + payload := []string{ "--environment=" + projectContext.DefaultEnv, "--interval=1m", @@ -65,29 +324,32 @@ func ScalingInstance(projectContext entity.ProjectGlobal) { usage.Values = append(usage.Values, value) usages[name] = usage } +} - // Get instance - log.Println("Get number of instance of project...") - payload = []string{ +func GetContainersUsage(projectContext entity.ProjectGlobal, usages map[string]UsageApp) { + log.Println("Get size of containers and number of instance of project...") + + payload := []string{ "--environment=" + projectContext.DefaultEnv, "--format=csv", - "--columns=service,instance_count", + "--columns=service,instance_count,cpu", "--no-header", "--no-interaction", "--yes", } - output, err = utils.CallCLIString(projectContext, "resources:get", payload...) + output, err := utils.CallCLIString(projectContext, "resources:get", payload...) if err != nil { log.Fatalf("command execution failed: %s", err) } // Parse output - apps = strings.Split(output, "\n") + apps := strings.Split(output, "\n") for _, app_str := range apps[:len(apps)-1] { app_raw := strings.Split(app_str, ",") // Normalize name := app_raw[0] + idProfil, _ := strconv.ParseFloat(app_raw[2], 64) instance, err := strconv.Atoi(app_raw[1]) if err != nil { break @@ -102,14 +364,18 @@ func ScalingInstance(projectContext entity.ProjectGlobal) { // Assign usage.InstanceOld = instance + usage.IdProfileOld = idProfil usages[name] = usage } } +} + +func RemoveUnscaleContainers(projectContext entity.ProjectGlobal, usages map[string]UsageApp) { if !app.ArgsS.IncludeServices { // Get Services log.Println("Dectect available services of project... (to exclude)") - payload = []string{ + payload := []string{ "--environment=" + projectContext.DefaultEnv, "--format=csv", "--columns=name", @@ -117,7 +383,7 @@ func ScalingInstance(projectContext entity.ProjectGlobal) { "--no-interaction", "--yes", } - output, err = utils.CallCLIString(projectContext, "service:list", payload...) + output, err := utils.CallCLIString(projectContext, "service:list", payload...) if err != nil { log.Fatalf("command execution failed: %s", err) } @@ -129,85 +395,59 @@ func ScalingInstance(projectContext entity.ProjectGlobal) { } } - // Compute - log.Println("Compute trend...") - for _, usage := range usages { + if strings.Contains(usage.Name, "---internal---storage") { + delete(usages, usage.Name) + } - //// Algo Threshold-limit - lastMetric := usage.Values[len(usage.Values)-1] + //if (usage.Name) + } +} - // CPU case - if lastMetric.Cpu > app.ArgsS.CpuUsageMin && usage.InstanceOld < app.ArgsS.HostCountMax { - instanceProposal := int(math.Ceil(float64(usage.InstanceOld) * (lastMetric.Cpu / app.ArgsS.CpuUsageMin))) - if usage.InstanceNew < instanceProposal { - usage.InstanceNew = instanceProposal - if usage.InstanceNew != usage.InstanceOld { - log.Printf("Upscale instance %v from %v to %v ! (Cpu: %v > %v)", usage.Name, usage.InstanceOld, usage.InstanceNew, lastMetric.Cpu, app.ArgsS.CpuUsageMin) - usage.Update = true - } - } - usages[usage.Name] = usage - } else if lastMetric.Cpu < app.ArgsS.CpuUsageMax && usage.InstanceOld > app.ArgsS.HostCountMin { - instanceProposal := int(math.Ceil(float64(usage.InstanceOld) * (lastMetric.Cpu / app.ArgsS.CpuUsageMax))) - if usage.InstanceNew < instanceProposal { - usage.InstanceNew = instanceProposal - if usage.InstanceNew != usage.InstanceOld { - log.Printf("Downscale instance %v from %v to %v ! (Cpu: %v < %v)", usage.Name, usage.InstanceOld, usage.InstanceNew, lastMetric.Cpu, app.ArgsS.CpuUsageMax) - usage.Update = true - } - } - usages[usage.Name] = usage - } +func SetResources(projectContext entity.ProjectGlobal, usages map[string]UsageApp) { + log.Println("Set new number of instance...") - // Mem case - if lastMetric.Mem > app.ArgsS.MemUsageMin && usage.InstanceOld < app.ArgsS.HostCountMax { - instanceProposal := int(math.Ceil(float64(usage.InstanceOld) * (lastMetric.Mem / app.ArgsS.MemUsageMin))) - if usage.InstanceNew < instanceProposal { - usage.InstanceNew = instanceProposal - if usage.InstanceNew != usage.InstanceOld { - log.Printf("Upscale instance %v from %v to %v ! (Mem: %v > %v)", usage.Name, usage.InstanceOld, usage.InstanceNew, lastMetric.Mem, app.ArgsS.MemUsageMin) - usage.Update = true - } - } - usages[usage.Name] = usage - } else if lastMetric.Mem < app.ArgsS.MemUsageMax && usage.InstanceOld > app.ArgsS.HostCountMin { - instanceProposal := int(math.Ceil(float64(usage.InstanceOld) * (lastMetric.Mem / app.ArgsS.MemUsageMax))) - if usage.InstanceNew < instanceProposal { - usage.InstanceNew = instanceProposal - if usage.InstanceNew != usage.InstanceOld { - log.Printf("Downscale instance %v from %v to %v ! (Mem: %v > %v)", usage.Name, usage.InstanceOld, usage.InstanceNew, lastMetric.Mem, app.ArgsS.MemUsageMax) - usage.Update = true - } + var updateInstance string + var updateContainer string + for _, usage := range usages { + if usage.InstanceUpdate { + if updateInstance != "" { + updateInstance += "," } - usages[usage.Name] = usage + updateInstance += usage.Name + updateInstance += ":" + updateInstance += strconv.Itoa(usage.InstanceNew) } } - // Reassign resource - log.Println("Set new number of instance...") - var update string + log.Println("Set new size of containers...") for _, usage := range usages { - if usage.Update { - if update != "" { - update += "," + if usage.IdProfileUpdate { + if updateContainer != "" { + updateContainer += "," } - update += usage.Name - update += ":" - update += strconv.Itoa(usage.InstanceNew) + updateContainer += usage.Name + updateContainer += ":" + updateContainer += usage.IdProfileNew.Id } } - if update != "" { - payload = []string{ + if updateInstance != "" || updateContainer != "" { + log.Println("Apply new resources...") + payload := []string{ "--environment=" + projectContext.DefaultEnv, "--no-interaction", "--yes", "--no-wait", - "--count", - update, } - output, err = utils.CallCLIString(projectContext, "resources:set", payload...) + if updateInstance != "" { + payload = append(payload, "--count", updateInstance) + } + if updateContainer != "" { + payload = append(payload, "--size", updateContainer) + } + + output, err := utils.CallCLIString(projectContext, "resources:set", payload...) if err != nil { log.Fatalf("command execution failed: %s", err) }