Add autoscaling (only up) for client (#249)

e2b-dev · Feb 10, 2025 · 0cb092e · 0cb092e
2 parents 025febf + 0eb585a
commit 0cb092e
Show file tree

Hide file tree

Showing 8 changed files with 47 additions and 9 deletions.
diff --git a/.env.template b/.env.template
@@ -17,6 +17,8 @@ TERRAFORM_STATE_BUCKET=
 CLIENT_MACHINE_TYPE=
 # e.g. 1
 CLIENT_CLUSTER_SIZE=
+# Max number of additional instances if the CPU usage is above 80%, e.g. 0
+CLIENT_CLUSTER_AUTO_SCALING_MAX=
 
 # This is the nomad and consul server (only for scheduling and service discovery)
 # eg e2-standard-2

diff --git a/Makefile b/Makefile
@@ -5,10 +5,12 @@ ENV_FILE := $(PWD)/.env.${ENV}
 
 OTEL_TRACING_PRINT ?= false
 EXCLUDE_GITHUB ?= 1
-TEMPLATE_BUCKET_LOCATION := $(GCP_REGION)
+TEMPLATE_BUCKET_LOCATION ?= $(GCP_REGION)
+CLIENT_CLUSTER_AUTO_SCALING_MAX ?= 0
 
 tf_vars := TF_VAR_client_machine_type=$(CLIENT_MACHINE_TYPE) \
 	TF_VAR_client_cluster_size=$(CLIENT_CLUSTER_SIZE) \
+	TF_VAR_client_cluster_auto_scaling_max=$(CLIENT_CLUSTER_AUTO_SCALING_MAX) \
 	TF_VAR_api_machine_type=$(API_MACHINE_TYPE) \
 	TF_VAR_api_cluster_size=$(API_CLUSTER_SIZE) \
 	TF_VAR_server_machine_type=$(SERVER_MACHINE_TYPE) \

diff --git a/main.tf b/main.tf
@@ -107,9 +107,10 @@ module "cluster" {
   gcp_zone                         = var.gcp_zone
   google_service_account_key       = module.init.google_service_account_key
 
-  server_cluster_size = var.server_cluster_size
-  client_cluster_size = var.client_cluster_size
-  api_cluster_size    = var.api_cluster_size
+  server_cluster_size             = var.server_cluster_size
+  client_cluster_size             = var.client_cluster_size
+  client_cluster_auto_scaling_max = var.client_cluster_auto_scaling_max
+  api_cluster_size                = var.api_cluster_size
 
   server_machine_type = var.server_machine_type
   client_machine_type = var.client_machine_type

diff --git a/packages/cluster/client/main.tf b/packages/cluster/client/main.tf
@@ -15,6 +15,24 @@ resource "google_compute_health_check" "nomad_check" {
   }
 }
 
+resource "google_compute_autoscaler" "default" {
+  provider = google-beta
+
+  name   = "${var.cluster_name}-autoscaler"
+  zone   = var.gcp_zone
+  target = google_compute_instance_group_manager.client_cluster.id
+
+  autoscaling_policy {
+    max_replicas    = var.cluster_size + var.cluster_auto_scaling_max
+    min_replicas    = var.cluster_size
+    cooldown_period = 240
+    mode            = "ONLY_SCALE_OUT"
+
+    cpu_utilization {
+      target = 0.8
+    }
+  }
+}
 
 resource "google_compute_instance_group_manager" "client_cluster" {
   name = "${var.cluster_name}-ig"
@@ -52,7 +70,6 @@ resource "google_compute_instance_group_manager" "client_cluster" {
   }
 
   base_instance_name = var.cluster_name
-  target_size        = var.cluster_size
   target_pools       = var.instance_group_target_pools
 
   depends_on = [

diff --git a/packages/cluster/client/variables.tf b/packages/cluster/client/variables.tf
@@ -28,6 +28,12 @@ variable "cluster_size" {
   type        = number
 }
 
+
+variable "cluster_auto_scaling_max" {
+  description = "The maximum number of additional nodes to have in the Nomad cluster based on the load."
+  type        = number
+}
+
 variable "image_family" {
   description = "The source image family used to create the boot disk for a Vault node. Only images based on Ubuntu 16.04 or 18.04 LTS are supported at this time."
   type        = string

diff --git a/packages/cluster/main.tf b/packages/cluster/main.tf
@@ -130,10 +130,11 @@ module "client_cluster" {
 
   environment = var.environment
 
-  cluster_name     = "${var.prefix}${var.client_cluster_name}"
-  cluster_size     = var.client_cluster_size
-  cluster_tag_name = var.cluster_tag_name
-  gcp_zone         = var.gcp_zone
+  cluster_name             = "${var.prefix}${var.client_cluster_name}"
+  cluster_auto_scaling_max = var.client_cluster_auto_scaling_max
+  cluster_size             = var.client_cluster_size
+  cluster_tag_name         = var.cluster_tag_name
+  gcp_zone                 = var.gcp_zone
 
   machine_type = var.client_machine_type
   image_family = var.client_image_family

diff --git a/packages/cluster/variables.tf b/packages/cluster/variables.tf
@@ -93,6 +93,11 @@ variable "client_cluster_size" {
   type = number
 }
 
+
+variable "client_cluster_auto_scaling_max" {
+  type = number
+}
+
 variable "client_machine_type" {
   type = string
 }

diff --git a/variables.tf b/variables.tf
@@ -24,6 +24,10 @@ variable "client_cluster_size" {
   type = number
 }
 
+variable "client_cluster_auto_scaling_max" {
+  type = number
+}
+
 variable "client_machine_type" {
   type = string
 }