Separate GitHub tf from infra terraform e2b 1520

Makefile

@@ -2,7 +2,6 @@ ENV := $(shell cat .last_used_env || echo "not-set")
 -include .env.${ENV}
 
 OTEL_TRACING_PRINT ?= false
-EXCLUDE_GITHUB ?= 1
 TEMPLATE_BUCKET_LOCATION := $(GCP_REGION)
 
 tf_vars := TF_VAR_client_machine_type=$(CLIENT_MACHINE_TYPE) \
@@ -22,12 +21,6 @@ tf_vars := TF_VAR_client_machine_type=$(CLIENT_MACHINE_TYPE) \
 	TF_VAR_template_bucket_name=$(TEMPLATE_BUCKET_NAME) \
 	TF_VAR_template_bucket_location=$(TEMPLATE_BUCKET_LOCATION)
 
-ifeq ($(EXCLUDE_GITHUB),1)
-	ALL_MODULES := $(shell cat main.tf | grep "^module" | awk '{print $$2}' | grep -v -e "github_tf")
-else
-	ALL_MODULES := $(shell cat main.tf | grep "^module" | awk '{print $$2}')
-endif
-
 # Login for Packer and Docker (uses gcloud user creds)
 # Login for Terraform (uses application default creds)
 .PHONY: login-gcloud
@@ -51,14 +44,12 @@ init:
 plan:
 	@ printf "Planning Terraform for env: `tput setaf 2``tput bold`$(ENV)`tput sgr0`\n\n"
 	terraform fmt -recursive
-	$(eval TARGET := $(shell echo $(ALL_MODULES) | tr ' ' '\n' | awk '{print "-target=module." $$0 ""}' | xargs))
-	$(tf_vars) terraform plan -out=.tfplan.$(ENV) -compact-warnings -detailed-exitcode $(TARGET)
+	$(tf_vars) terraform plan -out=.tfplan.$(ENV) -compact-warnings -detailed-exitcode
 
 .PHONY: plan-only-jobs
 plan-only-jobs:
 	@ printf "Planning Terraform for env: `tput setaf 2``tput bold`$(ENV)`tput sgr0`\n\n"
 	terraform fmt -recursive
-	$(eval TARGET := $(shell echo $(ALL_MODULES) | tr ' ' '\n' | awk '{print "-target=module." $$0 ""}' | xargs))
 	$(tf_vars) terraform plan -out=.tfplan.$(ENV) -compact-warnings -detailed-exitcode -target=module.nomad
 
 
@@ -78,7 +69,7 @@ apply:
 .PHONY: plan-without-jobs
 plan-without-jobs:
 	@ printf "Planning Terraform for env: `tput setaf 2``tput bold`$(ENV)`tput sgr0`\n\n"
-	$(eval TARGET := $(shell echo $(ALL_MODULES) | tr ' ' '\n' | grep -v -e "nomad" | awk '{print "-target=module." $$0 ""}' | xargs))
+	$(eval TARGET := $(shell cat main.tf | grep "^module" | awk '{print $$2}' | tr ' ' '\n' | grep -v -e "nomad" | awk '{print "-target=module." $$0 ""}' | xargs))
 	$(tf_vars) \
 	terraform plan \
 	-out=.tfplan.$(ENV) \

github-tf/Makefile

@@ -0,0 +1,47 @@
+ENV := $(shell cat ../.last_used_env || echo "not-set")
+-include ../.env.${ENV}
+
+tf_vars := 	TF_VAR_gcp_project_id=$(GCP_PROJECT_ID) \
+	TF_VAR_gcp_region=$(GCP_REGION) \
+	TF_VAR_gcp_zone=$(GCP_ZONE) \
+	TF_VAR_domain_name=$(DOMAIN_NAME) \
+	TF_VAR_prefix=$(PREFIX) \
+	TF_VAR_terraform_state_bucket=$(TERRAFORM_STATE_BUCKET) \
+	TF_VAR_environment=$(TERRAFORM_ENVIRONMENT) \
+	TF_VAR_template_bucket_name=$(TEMPLATE_BUCKET_NAME) \
+	TF_VAR_template_bucket_location=$(TEMPLATE_BUCKET_LOCATION)
+
+
+
+.PHONY: init
+init:
+	@ printf "Initializing Terraform for env: `tput setaf 2``tput bold`$(ENV)`tput sgr0`\n\n"
+	../scripts/confirm.sh $(ENV)
+	terraform init -input=false -backend-config="bucket=${TERRAFORM_STATE_BUCKET}"
+
+.PHONY: plan
+plan:
+	@ printf "Planning Terraform for env: `tput setaf 2``tput bold`$(ENV)`tput sgr0`\n\n"
+	terraform fmt -recursive
+	$(tf_vars) terraform plan -out=.tfplan.$(ENV) -compact-warnings -detailed-exitcode
+
+.PHONY: apply
+apply:
+	@ printf "Applying Terraform for env: `tput setaf 2``tput bold`$(ENV)`tput sgr0`\n\n"
+	../scripts/confirm.sh $(ENV)
+	$(tf_vars) \
+	terraform apply \
+	-auto-approve \
+	-input=false \
+	-compact-warnings \
+	-parallelism=20 \
+	.tfplan.$(ENV)
+	@ rm .tfplan.$(ENV)
+
+
+# Shortcut to importing resources into Terraform state (e.g. after creating resources manually or switching between different branches for the same environment)
+.PHONY: import
+import:
+	@ printf "Importing resources for env: `tput setaf 2``tput bold`$(ENV)`tput sgr0`\n\n"
+	./scripts/confirm.sh $(ENV)
+	$(tf_vars) terraform import $(TARGET) $(ID)

github-tf/main.tf

@@ -1,12 +1,30 @@
 terraform {
+  required_version = ">= 1.5.0, < 1.6.0"
+  backend "gcs" {
+    prefix = "terraform/github-cicd/state"
+  }
   required_providers {
     github = {
       source  = "integrations/github"
       version = "5.42.0"
     }
+    google = {
+      source  = "hashicorp/google"
+      version = "5.31.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "3.5.1"
+    }
   }
 }
 
+provider "google" {
+  project = var.gcp_project_id
+  region  = var.gcp_region
+  zone    = var.gcp_zone
+}
+
 resource "google_secret_manager_secret" "github_token" {
   secret_id = "${var.prefix}github-repo-token"
 
@@ -147,13 +165,13 @@ resource "google_storage_bucket_iam_member" "public_builds_bucket_iam" {
 }
 
 resource "google_storage_bucket_iam_member" "fc_kernels_bucket_iam" {
-  bucket = var.kernel_bucket
+  bucket = "${var.gcp_project_id}-fc-kernels"
   role   = "roles/storage.objectAdmin"
   member = "serviceAccount:${google_service_account.github_action_service_account.email}"
 }
 
 resource "google_storage_bucket_iam_member" "fc_versions_bucket_iam" {
-  bucket = var.fc_versions_bucket
+  bucket = "${var.gcp_project_id}-fc-versions"
   role   = "roles/storage.objectAdmin"
   member = "serviceAccount:${google_service_account.github_action_service_account.email}"
 }

github-tf/variables.tf

@@ -16,11 +16,13 @@ variable "gcp_zone" {
 variable "github_organization" {
   description = "The name of the github organization"
   type        = string
+  default     = "e2b-dev"
 }
 
 variable "github_repository" {
   description = "The name of the repository"
   type        = string
+  default     = "infra"
 }
 
 variable "prefix" {
@@ -33,16 +35,6 @@ variable "terraform_state_bucket" {
   type        = string
 }
 
-variable "fc_versions_bucket" {
-  description = "The name of the bucket to store build fc versions"
-  type        = string
-}
-
-variable "kernel_bucket" {
-  description = "The name of the bucket to store built kernels"
-  type        = string
-}
-
 variable "domain_name" {
   type = string
 }

main.tf

@@ -24,10 +24,6 @@ terraform {
       source  = "hashicorp/nomad"
       version = "2.1.0"
     }
-    github = {
-      source  = "integrations/github"
-      version = "5.42.0"
-    }
     random = {
       source  = "hashicorp/random"
       version = "3.5.1"
@@ -78,24 +74,6 @@ module "buckets" {
   labels = var.labels
 }
 
-module "github_tf" {
-  source = "./github-tf"
-
-  gcp_project_id = var.gcp_project_id
-  gcp_region     = var.gcp_region
-  gcp_zone       = var.gcp_zone
-
-  github_organization = var.github_organization
-  github_repository   = var.github_repository
-
-  domain_name            = var.domain_name
-  terraform_state_bucket = var.terraform_state_bucket
-  kernel_bucket          = module.buckets.fc_kernels_bucket_name
-  fc_versions_bucket     = module.buckets.fc_versions_bucket_name
-
-  prefix = var.prefix
-}
-
 module "cluster" {
   source = "./packages/cluster"
 

variables.tf

@@ -163,16 +163,6 @@ variable "otel_tracing_print" {
   default     = false
 }
 
-variable "github_organization" {
-  type    = string
-  default = "e2b-dev"
-}
-
-variable "github_repository" {
-  type    = string
-  default = "infra"
-}
-
 variable "domain_name" {
   type        = string
   description = "The domain name where e2b will run"