From 3b7381653b760aebd937073504183caa3ce1956d Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Mon, 14 Aug 2023 17:51:02 +0200 Subject: [PATCH 1/7] Update for deployment in a new project [WIP] --- .github/disabled-workflows/devbookd.yml | 6 +- .github/disabled-workflows/fc-env.yml | 6 +- .../firecracker-task-driver.yml | 9 +- .gitignore | 2 + main.tf | 128 +++++++++--------- packages/api/Makefile | 4 +- packages/api/go-templates/fc-build-envs.hcl | 4 +- packages/api/go-templates/fc-update-envs.hcl | 2 +- .../api/go-templates/fc-use-prebuilt-envs.hcl | 2 +- packages/api/variables.tf | 2 +- packages/cluster-disk-image/main.pkr.hcl | 10 +- packages/cluster-disk-image/variables.pkr.hcl | 4 +- packages/devbookd/upload-local.sh | 2 +- packages/devbookd/upload.sh | 2 +- packages/fc-env/publish.sh | 2 +- packages/firecracker-task-driver/Makefile | 2 +- packages/firecracker-task-driver/upload.sh | 2 +- packages/telemetry/logs-collector.hcl | 4 +- packages/telemetry/main.tf | 12 +- packages/telemetry/variables.tf | 2 +- variables.tf | 4 +- 21 files changed, 106 insertions(+), 105 deletions(-) diff --git a/.github/disabled-workflows/devbookd.yml b/.github/disabled-workflows/devbookd.yml index 9842e88981..00718a46a9 100644 --- a/.github/disabled-workflows/devbookd.yml +++ b/.github/disabled-workflows/devbookd.yml @@ -42,7 +42,7 @@ jobs: service_account: ${{ secrets.service_account_email }} - name: Upload devbookd - uses: 'google-github-actions/upload-cloud-storage@v1' + uses: "google-github-actions/upload-cloud-storage@v1" with: - path: './packages/devbookd/dist/devbookd_linux_amd64_v1/devbookd' - destination: 'devbook-environment-pipeline' + path: "./packages/devbookd/dist/devbookd_linux_amd64_v1/devbookd" + destination: "e2b-fc-env-pipeline" diff --git a/.github/disabled-workflows/fc-env.yml b/.github/disabled-workflows/fc-env.yml index 8785db5e6d..f0167488db 100644 --- a/.github/disabled-workflows/fc-env.yml +++ b/.github/disabled-workflows/fc-env.yml @@ -27,7 +27,7 @@ jobs: service_account: ${{ secrets.service_account_email }} - name: Upload env pipelines - uses: 'google-github-actions/upload-cloud-storage@v1' + uses: "google-github-actions/upload-cloud-storage@v1" with: - path: './packages/fc-env/env.tar.gz' - destination: 'devbook-environment-pipeline' + path: "./packages/fc-env/env.tar.gz" + destination: "e2b-fc-env-pipeline" diff --git a/.github/disabled-workflows/firecracker-task-driver.yml b/.github/disabled-workflows/firecracker-task-driver.yml index 6918e3bb8c..883608bb45 100644 --- a/.github/disabled-workflows/firecracker-task-driver.yml +++ b/.github/disabled-workflows/firecracker-task-driver.yml @@ -16,14 +16,13 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - - name: Set up Go uses: actions/setup-go@v3 with: go-version-file: ./packages/firecracker-task-driver/go.mod cache: true cache-dependency-path: ./packages/firecracker-task-driver/go.sum - + - name: Download deps working-directory: ./packages/firecracker-task-driver run: go mod tidy @@ -39,7 +38,7 @@ jobs: service_account: ${{ secrets.service_account_email }} - name: Upload firecracker task driver - uses: 'google-github-actions/upload-cloud-storage@v1' + uses: "google-github-actions/upload-cloud-storage@v1" with: - path: './packages/firecracker-task-driver/bin/firecracker-task-driver' - destination: 'devbook-environment-pipeline' \ No newline at end of file + path: "./packages/firecracker-task-driver/bin/firecracker-task-driver" + destination: "e2b-fc-env-pipeline" diff --git a/.gitignore b/.gitignore index 3b0fde4cc2..72e1c5f58f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ core.3071 node_modules # Ignore Vale.sh styles directory vale-styles + +.env diff --git a/main.tf b/main.tf index d4fc4e1c6b..60ad5194e6 100644 --- a/main.tf +++ b/main.tf @@ -1,7 +1,7 @@ terraform { required_version = ">=1.1.9" backend "gcs" { - bucket = "devbook-terraform-state" + bucket = "e2b-terraform-state" prefix = "terraform/orchestration/state" } required_providers { @@ -57,84 +57,84 @@ data "google_secret_manager_secret_version" "consul_acl_token" { secret = "consul-secret-id" } -provider "consul" { - address = "http://${data.google_compute_global_address.orch_server_consul_ip.address}" - token = data.google_secret_manager_secret_version.consul_acl_token.secret_data -} +# provider "consul" { +# address = "http://${data.google_compute_global_address.orch_server_consul_ip.address}" +# token = data.google_secret_manager_secret_version.consul_acl_token.secret_data +# } -resource "consul_acl_policy" "agent" { - name = "agent" - rules = <<-RULE - key_prefix "" { - policy = "deny" - } - RULE -} +# resource "consul_acl_policy" "agent" { +# name = "agent" +# rules = <<-RULE +# key_prefix "" { +# policy = "deny" +# } +# RULE +# } -resource "consul_acl_token_policy_attachment" "attachment" { - token_id = "00000000-0000-0000-0000-000000000002" - policy = consul_acl_policy.agent.name -} +# resource "consul_acl_token_policy_attachment" "attachment" { +# token_id = "00000000-0000-0000-0000-000000000002" +# policy = consul_acl_policy.agent.name +# } -data "google_compute_global_address" "orch_server_ip" { - name = "orch-server-ip" -} +# data "google_compute_global_address" "orch_server_ip" { +# name = "orch-server-ip" +# } -data "google_secret_manager_secret_version" "nomad_acl_token" { - secret = "nomad-secret-id" -} +# data "google_secret_manager_secret_version" "nomad_acl_token" { +# secret = "nomad-secret-id" +# } -provider "nomad" { - address = "http://${data.google_compute_global_address.orch_server_ip.address}" - secret_id = data.google_secret_manager_secret_version.nomad_acl_token.secret_data -} +# provider "nomad" { +# address = "http://${data.google_compute_global_address.orch_server_ip.address}" +# secret_id = data.google_secret_manager_secret_version.nomad_acl_token.secret_data +# } -data "google_secret_manager_secret_version" "lightstep_api_key" { - secret = "lightstep-api-key" -} +# data "google_secret_manager_secret_version" "lightstep_api_key" { +# secret = "lightstep-api-key" +# } -data "google_secret_manager_secret_version" "logtail_api_key" { - secret = "logtail-api-key" -} +# data "google_secret_manager_secret_version" "betterstack_logs_api_key" { +# secret = "betterstack-logs-api-key" +# } -module "telemetry" { - source = "./packages/telemetry" +# module "telemetry" { +# source = "./packages/telemetry" - logs_health_proxy_port = var.logs_health_proxy_port - logs_proxy_port = var.logs_proxy_port - lightstep_api_key = data.google_secret_manager_secret_version.lightstep_api_key.secret_data - logtail_api_key = data.google_secret_manager_secret_version.logtail_api_key.secret_data - gcp_zone = var.gcp_zone -} +# logs_health_proxy_port = var.logs_health_proxy_port +# logs_proxy_port = var.logs_proxy_port +# lightstep_api_key = data.google_secret_manager_secret_version.lightstep_api_key.secret_data +# betterstack_logs_api_key = data.google_secret_manager_secret_version.betterstack_logs_api_key.secret_data +# gcp_zone = var.gcp_zone +# } -module "session_proxy" { - source = "./packages/session-proxy" +# module "session_proxy" { +# source = "./packages/session-proxy" - client_cluster_size = var.client_cluster_size - gcp_zone = var.gcp_zone - session_proxy_service_name = var.session_proxy_service_name +# client_cluster_size = var.client_cluster_size +# gcp_zone = var.gcp_zone +# session_proxy_service_name = var.session_proxy_service_name - session_proxy_port = var.session_proxy_port -} +# session_proxy_port = var.session_proxy_port +# } -module "client_proxy" { - source = "./packages/client-proxy" +# module "client_proxy" { +# source = "./packages/client-proxy" - gcp_zone = var.gcp_zone - session_proxy_service_name = var.session_proxy_service_name +# gcp_zone = var.gcp_zone +# session_proxy_service_name = var.session_proxy_service_name - client_proxy_port = var.client_proxy_port - client_proxy_health_port = var.client_proxy_health_port -} +# client_proxy_port = var.client_proxy_port +# client_proxy_health_port = var.client_proxy_health_port +# } -module "api" { - source = "./packages/api" +# module "api" { +# source = "./packages/api" - gcp_zone = var.gcp_zone +# gcp_zone = var.gcp_zone - logs_proxy_address = "http://${module.cluster.logs_proxy_ip}" - nomad_address = "http://${module.cluster.server_proxy_ip}" - nomad_token = data.google_secret_manager_secret_version.nomad_acl_token.secret_data - consul_token = data.google_secret_manager_secret_version.consul_acl_token.secret_data - api_port = var.api_port -} +# logs_proxy_address = "http://${module.cluster.logs_proxy_ip}" +# nomad_address = "http://${module.cluster.server_proxy_ip}" +# nomad_token = data.google_secret_manager_secret_version.nomad_acl_token.secret_data +# consul_token = data.google_secret_manager_secret_version.consul_acl_token.secret_data +# api_port = var.api_port +# } diff --git a/packages/api/Makefile b/packages/api/Makefile index 1a6e5daf75..3520cc2d77 100644 --- a/packages/api/Makefile +++ b/packages/api/Makefile @@ -1,6 +1,6 @@ openapi := ../../openapi.yml -GCP_PROJECT := devbookhq +GCP_PROJECT := e2b-prod IMAGE := orchestration/api generate: @@ -9,7 +9,7 @@ generate: oapi-codegen -old-config-style -generate spec --package api $(openapi) > internal/api/spec.gen.go build: - CGO_ENABLED=0 go build -o bin/api . + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/api . build-debug: CGO_ENABLED=1 go build -race -gcflags=all="-N -l" -o bin/api . diff --git a/packages/api/go-templates/fc-build-envs.hcl b/packages/api/go-templates/fc-build-envs.hcl index 19831b9484..f6eea0451e 100644 --- a/packages/api/go-templates/fc-build-envs.hcl +++ b/packages/api/go-templates/fc-build-envs.hcl @@ -29,12 +29,12 @@ job "{{ .JobName }}/{{ .CodeSnippetID }}" { driver = "raw_exec" artifact { - source = "https://storage.googleapis.com/devbook-environment-pipeline/env.tar.gz" + source = "https://storage.googleapis.com/e2b-fc-env-pipeline/env.tar.gz" destination = "local" } artifact { - source = "https://storage.googleapis.com/devbook-environment-pipeline/devbookd" + source = "https://storage.googleapis.com/e2b-fc-env-pipeline/devbookd" destination = "local/devbookd" mode = "file" } diff --git a/packages/api/go-templates/fc-update-envs.hcl b/packages/api/go-templates/fc-update-envs.hcl index 065265cc60..db774bbfb7 100644 --- a/packages/api/go-templates/fc-update-envs.hcl +++ b/packages/api/go-templates/fc-update-envs.hcl @@ -30,7 +30,7 @@ job "{{ .JobName }}/{{ .CodeSnippetID }}" { } artifact { - source = "https://storage.googleapis.com/devbook-environment-pipeline/env.tar.gz" + source = "https://storage.googleapis.com/e2b-fc-env-pipeline/env.tar.gz" destination = "local" } diff --git a/packages/api/go-templates/fc-use-prebuilt-envs.hcl b/packages/api/go-templates/fc-use-prebuilt-envs.hcl index f58ddc8b0a..1e71d83509 100644 --- a/packages/api/go-templates/fc-use-prebuilt-envs.hcl +++ b/packages/api/go-templates/fc-use-prebuilt-envs.hcl @@ -30,7 +30,7 @@ job "{{ .JobName }}/{{ .CodeSnippetID }}" { } artifact { - source = "https://storage.googleapis.com/devbook-environment-pipeline/env.tar.gz" + source = "https://storage.googleapis.com/e2b-fc-env-pipeline/env.tar.gz" destination = "local" } diff --git a/packages/api/variables.tf b/packages/api/variables.tf index 6f013db78c..ba1c819ada 100644 --- a/packages/api/variables.tf +++ b/packages/api/variables.tf @@ -25,7 +25,7 @@ variable "api_port" { variable "image_name" { type = string - default = "us-central1-docker.pkg.dev/devbookhq/orchestration/api" + default = "us-central1-docker.pkg.dev/e2b-prod/orchestration/api" } variable "logs_proxy_address" { diff --git a/packages/cluster-disk-image/main.pkr.hcl b/packages/cluster-disk-image/main.pkr.hcl index ffd500ca38..64a3b7f20a 100644 --- a/packages/cluster-disk-image/main.pkr.hcl +++ b/packages/cluster-disk-image/main.pkr.hcl @@ -1,5 +1,5 @@ packer { - required_version = "1.8.4" + required_version = ">=1.8.4" required_plugins { googlecompute = { version = "1.0.16" @@ -14,7 +14,7 @@ source "googlecompute" "orch" { # TODO: Overwrite the image instead of creating timestamped images every time we build its image_name = "orch-${formatdate("YYYY-MM-DD-hh-mm-ss", timestamp())}" project_id = var.gcp_project_id - source_image_family = "ubuntu-2004-lts" + source_image_family = "ubuntu-2204-lts" ssh_username = "ubuntu" zone = var.gcp_zone disk_size = 10 @@ -52,7 +52,7 @@ build { provisioner "shell" { inline = [ "sudo apt-get update", - "sudo apt-get install -y docker.io=20.10.12-0ubuntu2~20.04.1", + "sudo apt-get install -y docker.io", "sudo systemctl start docker", "sudo usermod -aG docker $USER", ] @@ -90,7 +90,7 @@ build { provisioner "shell" { inline = [ "sudo mkdir -p /opt/nomad/plugins", - "sudo curl https://storage.googleapis.com/devbook-environment-pipeline/firecracker-task-driver -o /opt/nomad/plugins/firecracker-task-driver", + "sudo curl https://storage.googleapis.com/e2b-fc-env-pipeline/firecracker-task-driver -o /opt/nomad/plugins/firecracker-task-driver", "sudo chmod +x /opt/nomad/plugins/firecracker-task-driver", ] } @@ -98,7 +98,7 @@ build { provisioner "shell" { inline = [ "sudo mkdir -p /fc-vm", - "sudo curl https://storage.googleapis.com/devbook-snapshot/vmlinux.bin -o /fc-vm/vmlinux.bin", + "sudo curl https://storage.googleapis.com/e2b-fc-env-pipeline/vmlinux.bin -o /fc-vm/vmlinux.bin", ] } } diff --git a/packages/cluster-disk-image/variables.pkr.hcl b/packages/cluster-disk-image/variables.pkr.hcl index 1fd898a5f1..d3a82f8a54 100644 --- a/packages/cluster-disk-image/variables.pkr.hcl +++ b/packages/cluster-disk-image/variables.pkr.hcl @@ -1,11 +1,11 @@ variable "gcp_project_id" { type = string - default = "devbookhq" + default = "e2b-prod" } variable "gcp_zone" { type = string - default = "us-central1-a" + default = "us-central1-c" } variable "consul_version" { diff --git a/packages/devbookd/upload-local.sh b/packages/devbookd/upload-local.sh index 0f00ed870c..b3b7d4ddca 100755 --- a/packages/devbookd/upload-local.sh +++ b/packages/devbookd/upload-local.sh @@ -3,4 +3,4 @@ set -euo pipefail gsutil -h "Cache-Control:no-cache, max-age=0" \ - cp bin/devbookd gs://devbook-environment-pipeline/devbookd + cp bin/devbookd gs://e2b-fc-env-pipeline/devbookd diff --git a/packages/devbookd/upload.sh b/packages/devbookd/upload.sh index 0195da9aae..1d875c259a 100755 --- a/packages/devbookd/upload.sh +++ b/packages/devbookd/upload.sh @@ -3,4 +3,4 @@ set -euo pipefail gsutil -h "Cache-Control:no-cache, max-age=0" \ - cp dist/devbookd_linux_amd64_v1/devbookd gs://devbook-environment-pipeline/devbookd + cp dist/devbookd_linux_amd64_v1/devbookd gs://e2b-fc-env-pipeline/devbookd diff --git a/packages/fc-env/publish.sh b/packages/fc-env/publish.sh index 8683bb05e4..ba38c1a9cb 100755 --- a/packages/fc-env/publish.sh +++ b/packages/fc-env/publish.sh @@ -11,7 +11,7 @@ set -euo pipefail ./package.sh gsutil -h "Cache-Control:no-cache, max-age=0" \ - cp env.tar.gz gs://devbook-environment-pipeline + cp env.tar.gz gs://e2b-fc-env-pipeline rm env.tar.gz rm -rf env diff --git a/packages/firecracker-task-driver/Makefile b/packages/firecracker-task-driver/Makefile index 972affb27e..60a3ebf22c 100644 --- a/packages/firecracker-task-driver/Makefile +++ b/packages/firecracker-task-driver/Makefile @@ -4,7 +4,7 @@ upload: ./upload.sh build: - CGO_ENABLED=0 go build -o bin/ . + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/ . set-project-gcloud: gcloud config set project devbookhq diff --git a/packages/firecracker-task-driver/upload.sh b/packages/firecracker-task-driver/upload.sh index 1d1ee32c63..5a822bf7fc 100755 --- a/packages/firecracker-task-driver/upload.sh +++ b/packages/firecracker-task-driver/upload.sh @@ -3,4 +3,4 @@ set -euo pipefail gsutil -h "Cache-Control:no-cache, max-age=0" \ - cp bin/firecracker-task-driver gs://devbook-environment-pipeline/firecracker-task-driver + cp bin/firecracker-task-driver gs://e2b-fc-env-pipeline/firecracker-task-driver diff --git a/packages/telemetry/logs-collector.hcl b/packages/telemetry/logs-collector.hcl index 3ce3e53562..fc6ffc32b8 100644 --- a/packages/telemetry/logs-collector.hcl +++ b/packages/telemetry/logs-collector.hcl @@ -18,7 +18,7 @@ variable "logs_port_name" { type = string } -variable "logtail_api_key" { +variable "betterstack_logs_api_key" { type = string } @@ -114,7 +114,7 @@ inputs = [ "logtail_transform_Ng2hNptjHFG5TLHYJnMKdrAY" ] uri = "https://in.logtail.com/" encoding.codec = "json" auth.strategy = "bearer" -auth.token = "${var.logtail_api_key}" +auth.token = "${var.betterstack_logs_api_key}" EOH } } diff --git a/packages/telemetry/main.tf b/packages/telemetry/main.tf index 71db7e77af..802c46b9da 100644 --- a/packages/telemetry/main.tf +++ b/packages/telemetry/main.tf @@ -16,12 +16,12 @@ resource "nomad_job" "logs-collector" { hcl2 { enabled = true vars = { - logtail_api_key = var.logtail_api_key - logs_port_number = var.logs_proxy_port.port - logs_health_port_number = var.logs_health_proxy_port.port - logs_health_path = var.logs_health_proxy_port.health_path - logs_port_name = var.logs_proxy_port.name - gcp_zone = var.gcp_zone + betterstack_logs_api_key = var.betterstack_logs_api_key + logs_port_number = var.logs_proxy_port.port + logs_health_port_number = var.logs_health_proxy_port.port + logs_health_path = var.logs_health_proxy_port.health_path + logs_port_name = var.logs_proxy_port.name + gcp_zone = var.gcp_zone } } } diff --git a/packages/telemetry/variables.tf b/packages/telemetry/variables.tf index a3d929cf4d..b103f7dec4 100644 --- a/packages/telemetry/variables.tf +++ b/packages/telemetry/variables.tf @@ -6,7 +6,7 @@ variable "lightstep_api_key" { type = string } -variable "logtail_api_key" { +variable "betterstack_logs_api_key" { type = string } diff --git a/variables.tf b/variables.tf index 6b3abfe1e0..791077b02b 100644 --- a/variables.tf +++ b/variables.tf @@ -1,7 +1,7 @@ variable "gcp_project_id" { description = "The project to deploy the cluster in" type = string - default = "devbookhq" + default = "e2b-prod" } variable "gcp_region" { @@ -32,7 +32,7 @@ variable "client_cluster_size" { variable "client_machine_type" { type = string - default = "n1-standard-16" + default = "n1-standard-8" } variable "client_proxy_health_port" { From c1ba0dfd7740e1a3b6064d9f4bd5de0f2cbe108e Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Wed, 23 Aug 2023 10:39:10 +0200 Subject: [PATCH 2/7] Change devbook specific project setup; Improve API [WIP] --- .terraform.lock.hcl | 40 ----- DEV.md | 9 +- ENVS.md | 8 + Makefile | 2 +- openapi.yml | 167 ++++++------------ packages/api/go-templates/fc-build-envs.hcl | 2 +- packages/api/go-templates/fc-update-envs.hcl | 2 +- .../api/go-templates/fc-use-prebuilt-envs.hcl | 2 +- packages/client-proxy/client-proxy.hcl | 2 +- packages/cluster/client/main.tf | 10 +- packages/fc-env/build-env.sh | 2 +- packages/fc-env/update-env.sh | 29 ++- packages/fc-env/use-prebuilt-env.sh | 8 +- 13 files changed, 92 insertions(+), 191 deletions(-) create mode 100644 ENVS.md diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index d963cfec8e..96944be466 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -1,26 +1,6 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. -provider "registry.terraform.io/hashicorp/consul" { - version = "2.18.0" - hashes = [ - "h1:ABHNF3vYoRej5vLiiKR/XYgjhjubsjlVROssc555Vj8=", - "h1:GIbc9IimARunBuk+GdxgUX0cnZ2WwFW0IZMyT8E88B8=", - "zh:155f6899c8f1e0162169e4d09f426ea9a9738d74129f0ffc54b7570941c49cd4", - "zh:241ba909b387c349845b0371321af45ccbb00290edbe1ea0861ad8732951f718", - "zh:612f52be1fb5dd507b8064c67785313a531faad35873cba53f998f3766473335", - "zh:67fbd695381b9c5db83ffeeaec1bfdbcc477ccb1e9de1caff76a0186d4c85908", - "zh:7f88b151c9690a6addccbffe8484f0257344ef55424a9efb025dfddd052a4dc6", - "zh:8d954f3ffeb72b6c18cc5ae8c3189bb3a8cb66967b2106c7d0163009c12bba15", - "zh:913774c7eabc6e9078a1bd00347cc539b19a6f6b45dacbd21454dffdc9f4ae43", - "zh:9517558883f994649695643a6208079ed0445aaa0ac2dee69f88cb044d21c6c9", - "zh:a0211f596e35bd1b8d4bb9cda321cb1555a427a8d3f6724fe09893168fac9b7e", - "zh:a70eaa0a88f677f901855a4ab908ddcf961e4afaf3a0147b08faaead57b4fe07", - "zh:b02f5cd94ab236d988cfec531c56c199e3087803f1b2908a1a2b6da8a57b3751", - "zh:f3d3efac504c9484a025beb919d22b290aa6dbff256f6e86c1f8ce7817e077e5", - ] -} - provider "registry.terraform.io/hashicorp/google" { version = "3.90.1" constraints = "~> 3.53" @@ -63,26 +43,6 @@ provider "registry.terraform.io/hashicorp/google-beta" { ] } -provider "registry.terraform.io/hashicorp/nomad" { - version = "1.4.20" - hashes = [ - "h1:4IKRa9RKAwJ55lLQjgkOkXLnm+KV4T0bCH+ldNhRR14=", - "h1:M/QVXHPfeySejJZI3I8mBYrL/J9VsbnyF/dKIMlUhXo=", - "zh:02989edcebe724fc0aa873b22176fd20074c4f46295e728010711a8fc5dfa72c", - "zh:089ba7d19bcf5c6bab3f8b8c5920eb6d78c52cf79bb0c5dfeb411c600e7efcba", - "zh:235865a2182ca372bcbf440201a8b8cc0715ad5dbc4de893d99b6f32b5be53ab", - "zh:67ea718764f3f344ecc6e027d20c1327b86353c8064aa90da3ec12cec4a88954", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:8c68c540f0df4980568bdd688c2adec86eda62eb2de154e3db215b16de0a7ae0", - "zh:911969c63a69a733be57b96d54c5966c9424e1abec8d5f20038c8cef3a504c65", - "zh:a673c92ddc9d47e8d53dcb9b376f1adcb4543488202fc83a3e7eab8677530684", - "zh:a94a73eae89fd8c8ebf872013079be41161d3f293f4026c92d45c4c5667dd613", - "zh:db6b89f8b696040c0344f00928e4cf6e0a75034421ba14cdcd8a4d23bc865dce", - "zh:e512c0b1239e3d66b60d22c2b4de19fea288e492cde90dff9277cc475fd9dbbf", - "zh:ef6eccecbdef3bb8ce629cabfb5550c1db5c3e952943dda1786ef6cb470a8c23", - ] -} - provider "registry.terraform.io/kreuzwerker/docker" { version = "2.16.0" constraints = "2.16.0" diff --git a/DEV.md b/DEV.md index 47702f47e2..d9e12bfa50 100644 --- a/DEV.md +++ b/DEV.md @@ -2,8 +2,6 @@ ## Infra -- Clock drift - - https://github.com/firecracker-microvm/firecracker/blob/eb8de3ba1f7cb636d2aaa632fe96b234f3a302e6/FAQ.md#my-guest-wall-clock-is-drifting-how-can-i-fix-it - Change read/write file to allow other thatn utf-8 format so we don't break the files - Remove dnsmasq from cluster VM image if it is not needed - Remove need for provisioning during building env @@ -23,8 +21,8 @@ - Check if the kernel args we are using are what we want - Improve API (efficient calls Nomad calls, no polling, etc.) - Improve FC WS connection (subscriptions take additional calls, maybe we can improve that, etc.) -- Move to ubuntu - Update kernel version +- devbookd jsonrpc parameters could be objects instead of arrays (compatibility advantages) - Move to a better FS handling (filestore, fuse?) - Fix possible mutex problems in the firecracker task driver - Update devbookd in all envs automatically @@ -53,17 +51,14 @@ - Use the FC jailer properly - Rebuild only the changed templates on push - Update NodeJS version -- Remove top package-lock - Remove gitpod deps and mentions (makefile, gitpod yaml) - Remove devbook specific code from the repo (allow move to new org or on prem deploy) - Improve caching of the GH actions - Add instance sizes and other things as variables in the GH actions and remove them from codes - API URL should be configurable - add testing/staging env -- create proper monorepo for the packages (pnpm for Node) and improve release system - Add memory swap to Rust - OoM when compiling bigger programs - https://www.digitalocean.com/community/tutorials/how-to-add-swap-space-on-ubuntu-22-04 -- Should we use ubuntu 22 for envs? - Does ubuntu need the newer kernel? - Using driver with multiple kernels - Cloud Hypervisor insted of FC - https://github.com/cloud-hypervisor/cloud-hypervisor @@ -145,7 +140,7 @@ - add tests - support lower python versions - Check if stderr and stdout are ordered -- Myabe remove types (and timestamps?) from stdout/stderr +- Myabe remove types (and timestamps?) from stdout/stderr (they are already fully identified by the subscription) - Use specific version (nodejs20) for the templates - Using pathlib for paths? windows support? - Flush all stdout/err after killing process or terminal in devbookd and also wait for the Stdout/err in the SDK diff --git a/ENVS.md b/ENVS.md new file mode 100644 index 0000000000..124af07a39 --- /dev/null +++ b/ENVS.md @@ -0,0 +1,8 @@ +# Custom Envs + +- How to create a custom env from dockerfile if you want to use local files during the build step? + - It's either you build the dockerfile locally or you package and upload the files to our infra so it can be used during the build step. + + + +- Can we has the dockerfile+files+scripts/config to get the unique env hash that we can use to start identical envs in the future? diff --git a/Makefile b/Makefile index 10b3c230d0..0b2ec6c0a1 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ login-gcloud: .PHONY: login-gcloud-user login-gcloud-user: gcloud auth login - gcloud config set project devbookhq + gcloud config set project e2b-prod gcloud --quiet auth configure-docker us-central1-docker.pkg.dev .PHONY: init diff --git a/openapi.yml b/openapi.yml index 7e73c779bb..b3b2c49624 100644 --- a/openapi.yml +++ b/openapi.yml @@ -1,27 +1,26 @@ openapi: 3.0.0 info: - version: 1.0.0 - title: Devbook - description: Devbook API + version: 0.1.0 + title: E2B API servers: - - url: https://ondevbook.com/ + - url: https://api.e2b.dev/ description: API endpoint - - url: https://{sessionID}-{clientID}.ondevbook.com/ - description: Session endpoint without a specified port + - url: https://{instanceID}-{clientID}.api.e2b.dev/ + description: Instance endpoint without a specified port variables: - sessionID: - description: ID of the session - default: _sessionID + instanceID: + description: ID of the instance + default: _instanceID clientID: description: ID of the client default: _clientID - - url: https://{port}-{sessionID}-{clientID}.ondevbook.com/ - description: Session endpoint with a specificed port + - url: https://{port}-{intanceID}-{clientID}.api.e2b.dev/ + description: Instance endpoint with a specificed port variables: - sessionID: - description: ID of the session - default: _sessionID + instanceID: + description: ID of the instance + default: _instanceID clientID: description: ID of the client default: _clientID @@ -34,29 +33,23 @@ components: ApiKeyAuth: name: api_key type: apiKey - in: query + in: header parameters: - apiKeyOpt: + apiKey: name: api_key - in: query - required: false - schema: - type: string - apiKeyReq: - name: api_key - in: query + in: header required: true schema: type: string - codeSnippetID: - name: codeSnippetID + envID: + name: envID in: path required: true schema: type: string - sessionID: - name: sessionID + instanceID: + name: instanceID in: path required: true schema: @@ -83,85 +76,55 @@ components: $ref: "#/components/schemas/Error" schemas: - Template: - type: string - enum: - - Nodejs - - Go - - Bash - - Rust - - Python3 - - PHP - - Java - - Perl - - DotNET - - EnvironmentState: + EnvState: type: string enum: - Building - Failed - Done - Environment: + Env: required: - id properties: id: type: string - template: - type: string - title: + name: type: string - NewEnvironment: + NewEnv: required: - - template + - buildConfig properties: - title: + name: type: string - template: + buildConfig: type: string - EnvironmentStateUpdate: + EnvStateUpdate: required: - state properties: state: - $ref: "#/components/schemas/EnvironmentState" + $ref: "#/components/schemas/EnvState" - EnvironmentTitleUpdate: + EnvNameUpdate: properties: - title: + name: type: string - NewSession: - required: - - codeSnippetID - properties: - editEnabled: - type: boolean - default: false - description: Option determining if the session is a shared persistent edit session - codeSnippetID: - type: string - description: Identifier of a code snippet which which is the environment associated - Session: + Instance: required: - - sessionID + - envID + - instanceID - clientID - - editEnabled - - codeSnippetID properties: - codeSnippetID: + envID: type: string - description: Identifier of a code snippet which which is the environment associated - editEnabled: - type: boolean - description: Information if the session is a shared persistent edit session - sessionID: + description: Identifier of the environment + instanceID: type: string - description: Identifier of the session + description: Identifier of the instance clientID: type: string description: Identifier of the client @@ -180,10 +143,10 @@ components: description: Error tags: - - name: sessions - description: Managing VM sessions + - name: instances + description: Managing environment instances - name: envs - description: Environment for VM + description: Manage environments paths: /health: @@ -200,7 +163,7 @@ paths: description: List all sessions tags: [sessions] parameters: - - $ref: "#/components/parameters/apiKeyReq" + - $ref: "#/components/parameters/apiKey" responses: 200: description: Successfully returned all sessions @@ -210,16 +173,16 @@ paths: type: array items: allOf: - - $ref: "#/components/schemas/Session" + - $ref: "#/components/schemas/Instance" 401: $ref: "#/components/responses/401" 500: $ref: "#/components/responses/500" post: - description: Create a session on the server + description: Create an instance of the environment tags: [sessions] parameters: - - $ref: "#/components/parameters/apiKeyOpt" + - $ref: "#/components/parameters/apiKey" requestBody: required: true content: @@ -279,7 +242,7 @@ paths: description: List all environments tags: [envs] parameters: - - $ref: "#/components/parameters/apiKeyReq" + - $ref: "#/components/parameters/apiKey" responses: 200: description: Successfully returned all environments @@ -298,7 +261,7 @@ paths: description: Create a new environment tags: [envs] parameters: - - $ref: "#/components/parameters/apiKeyReq" + - $ref: "#/components/parameters/apiKey" requestBody: required: true content: @@ -319,29 +282,7 @@ paths: 500: $ref: "#/components/responses/500" - # This method will be replaced by POST /envs - /envs/{codeSnippetID}: - post: - description: Create a new env for a code snippet - tags: [envs] - parameters: - - $ref: "#/components/parameters/apiKeyReq" - - $ref: "#/components/parameters/codeSnippetID" - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/NewEnvironment" - responses: - 204: - description: Successfully created an environment - 400: - $ref: "#/components/responses/400" - 401: - $ref: "#/components/responses/401" - 500: - $ref: "#/components/responses/500" + /envs/{envID}: delete: description: Delete the code snippet environment tags: [envs] @@ -377,7 +318,7 @@ paths: 500: $ref: "#/components/responses/500" - /envs/{codeSnippetID}/state: + /envs/{envID}/state: put: description: Update the state of the environment tags: [envs] @@ -398,22 +339,22 @@ paths: 401: $ref: "#/components/responses/401" - /envs/{codeSnippetID}/title: + /envs/{envID}/title: put: description: Update the title of the environment tags: [envs] parameters: - - $ref: "#/components/parameters/apiKeyReq" - - $ref: "#/components/parameters/codeSnippetID" + - $ref: "#/components/parameters/apiKey" + - $ref: "#/components/parameters/envID" requestBody: required: true content: application/json: schema: - $ref: "#/components/schemas/EnvironmentTitleUpdate" + $ref: "#/components/schemas/EnvTitleUpdate" responses: 204: - description: Updated the title of the code snippet + description: Updated the title of the env 400: $ref: "#/components/responses/400" 401: diff --git a/packages/api/go-templates/fc-build-envs.hcl b/packages/api/go-templates/fc-build-envs.hcl index f6eea0451e..8eee2b763d 100644 --- a/packages/api/go-templates/fc-build-envs.hcl +++ b/packages/api/go-templates/fc-build-envs.hcl @@ -89,7 +89,7 @@ if [ -z "$API_KEY" ]; then exit 1 fi -API_URL="https://ondevbook.com" +API_URL="https://api.e2b.dev" ENVS_ENDPOINT="${API_URL}/envs/${CODE_SNIPPET_ID}/state?api_key=$API_KEY" # Main didn't finish successfully. diff --git a/packages/api/go-templates/fc-update-envs.hcl b/packages/api/go-templates/fc-update-envs.hcl index db774bbfb7..6ec9c00a45 100644 --- a/packages/api/go-templates/fc-update-envs.hcl +++ b/packages/api/go-templates/fc-update-envs.hcl @@ -83,7 +83,7 @@ if [ -z "$API_KEY" ]; then exit 1 fi -API_URL="https://ondevbook.com" +API_URL="https://api.e2b.dev" ENVS_ENDPOINT="${API_URL}/envs/${CODE_SNIPPET_ID}/state?api_key=$API_KEY" # Main didn't finish successfully. diff --git a/packages/api/go-templates/fc-use-prebuilt-envs.hcl b/packages/api/go-templates/fc-use-prebuilt-envs.hcl index 1e71d83509..00323f7f2d 100644 --- a/packages/api/go-templates/fc-use-prebuilt-envs.hcl +++ b/packages/api/go-templates/fc-use-prebuilt-envs.hcl @@ -81,7 +81,7 @@ if [ -z "$API_KEY" ]; then exit 1 fi -API_URL="https://ondevbook.com" +API_URL="https://api.e2b.dev" ENVS_ENDPOINT="${API_URL}/envs/${CODE_SNIPPET_ID}/state?api_key=$API_KEY" # Main didn't finish successfully. diff --git a/packages/client-proxy/client-proxy.hcl b/packages/client-proxy/client-proxy.hcl index 8f06e9e615..4274fffc31 100644 --- a/packages/client-proxy/client-proxy.hcl +++ b/packages/client-proxy/client-proxy.hcl @@ -92,7 +92,7 @@ server { [[ range service "session-proxy" ]] server { listen 3002; - server_name ~^(.+)-[[ index .ServiceMeta "Client" | sprig_substr 0 8 ]]\.ondevbook\.com$; + server_name ~^(.+)-[[ index .ServiceMeta "Client" | sprig_substr 0 8 ]]\.api\.e2b\.dev$; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; diff --git a/packages/cluster/client/main.tf b/packages/cluster/client/main.tf index 5abb7a702a..4dae3aebf6 100644 --- a/packages/cluster/client/main.tf +++ b/packages/cluster/client/main.tf @@ -130,8 +130,8 @@ data "google_compute_ssl_certificate" "session_certificate" { } # This should be SSL cert for usage withotu Cloudflare -data "google_compute_ssl_certificate" "ondevbook_certificate" { - name = "ondevbook" +data "google_compute_ssl_certificate" "api_certificate" { + name = "e2b-api" } resource "google_compute_url_map" "client_map" { @@ -139,12 +139,12 @@ resource "google_compute_url_map" "client_map" { default_service = module.gce_lb_http.backend_services["session"].self_link host_rule { - hosts = ["ondevbook.com"] + hosts = ["api.e2b.com"] path_matcher = "api-paths" } host_rule { - hosts = ["*.ondevbook.com"] + hosts = ["*.api.e2b.com"] path_matcher = "session-paths" } @@ -171,7 +171,7 @@ module "gce_lb_http" { address = data.google_compute_global_address.orch_client_ip.address ssl_certificates = [ data.google_compute_ssl_certificate.session_certificate.self_link, - data.google_compute_ssl_certificate.ondevbook_certificate.self_link, + data.google_compute_ssl_certificate.api_certificate.self_link, ] create_address = false use_ssl_certificates = true diff --git a/packages/fc-env/build-env.sh b/packages/fc-env/build-env.sh index b139c913f8..42cfedf07f 100755 --- a/packages/fc-env/build-env.sh +++ b/packages/fc-env/build-env.sh @@ -57,7 +57,7 @@ if [ -z "$DEVBOOKD" ]; then exit 1 fi -API_URL="https://ondevbook.com" +API_URL="https://api.e2b.dev" ENVS_ENDPOINT="${API_URL}/envs/${CODE_SNIPPET_ID}/state?api_key=$API_KEY" echo "==== Args ===========================================================================================" diff --git a/packages/fc-env/update-env.sh b/packages/fc-env/update-env.sh index 9157b10883..8a2182e2d5 100755 --- a/packages/fc-env/update-env.sh +++ b/packages/fc-env/update-env.sh @@ -46,7 +46,7 @@ if [ -z "$API_KEY" ]; then exit 1 fi -API_URL="https://ondevbook.com" +API_URL="https://api.e2b.dev" ENVS_ENDPOINT="${API_URL}/envs/${CODE_SNIPPET_ID}/state?api_key=$API_KEY" echo "==== Args ===========================================================================================" @@ -89,7 +89,7 @@ function mkdirs() { } function get_snapshot() { - EDIT_ID=`cat $EDIT_ID_PATH` + EDIT_ID=$(cat $EDIT_ID_PATH) EDIT_ID_DIR="$EDIT_DIR/$EDIT_ID" if [[ "$SESSION_ID" == "" ]]; then @@ -104,18 +104,18 @@ function get_snapshot() { echo "Running edit sessions rootfs path $SESSION_ROOTFS" curl --unix-socket $EDIT_FC_SOCK -i \ - -X PATCH 'http://localhost/vm' \ - -H 'Accept: application/json' \ - -H 'Content-Type: application/json' \ - -d '{ + -X PATCH 'http://localhost/vm' \ + -H 'Accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ "state": "Paused" }' curl --unix-socket $EDIT_FC_SOCK -i \ - -X PUT 'http://localhost/snapshot/create' \ - -H 'Accept: application/json' \ - -H 'Content-Type: application/json' \ - -d "{ + -X PUT 'http://localhost/snapshot/create' \ + -H 'Accept: application/json' \ + -H 'Content-Type: application/json' \ + -d "{ \"snapshot_type\": \"Full\", \"snapshot_path\": \"$BUILD_FC_SNAPFILE\", \"mem_file_path\": \"$BUILD_FC_MEMFILE\" @@ -124,10 +124,10 @@ function get_snapshot() { cp -pRd --reflink $SESSION_ROOTFS $BUILD_FC_ROOTFS curl --unix-socket $EDIT_FC_SOCK -i \ - -X PATCH 'http://localhost/vm' \ - -H 'Accept: application/json' \ - -H 'Content-Type: application/json' \ - -d '{ + -X PATCH 'http://localhost/vm' \ + -H 'Accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ "state": "Resumed" }' fi @@ -155,7 +155,6 @@ get_snapshot mv_env_files del_build_dir - touch ${ALLOC_DIR}/main-done echo "==== Output ===========================================================================================" diff --git a/packages/fc-env/use-prebuilt-env.sh b/packages/fc-env/use-prebuilt-env.sh index 327dedb6e5..d0f83749ff 100755 --- a/packages/fc-env/use-prebuilt-env.sh +++ b/packages/fc-env/use-prebuilt-env.sh @@ -40,7 +40,7 @@ if [ -z "$API_KEY" ]; then exit 1 fi -API_URL="https://ondevbook.com" +API_URL="https://api.e2b.dev" ENVS_ENDPOINT="${API_URL}/envs/${CODE_SNIPPET_ID}/state?api_key=$API_KEY" echo "==== Args ===========================================================================================" @@ -78,9 +78,9 @@ function mk_template_files() { if [ -f "$TEMPLATE_BUILD_ID" ]; then echo "$TEMPLATE_BUILD_ID exists - the template environemnt was rebuilt in this directory so we must copy the current build_id." - echo -n "${TEMPLATE}" > ${FINAL_TEMPLATE_ID_FILE} + echo -n "${TEMPLATE}" >${FINAL_TEMPLATE_ID_FILE} cp $TEMPLATE_BUILD_ID $FINAL_TEMPLATE_BUILD_ID_FILE - else + else echo "$TEMPLATE_BUILD_ID does not exist - the template environment uses build_id from its own template - we copy files pointing to its own template." cp $TEMPLATE_TEMPLATE_ID_FILE $FINAL_TEMPLATE_ID_FILE @@ -110,14 +110,12 @@ curl $ENVS_ENDPOINT \ \"state\": \"Building\" }" - mkdirs link_env_files mk_template_files touch ${ALLOC_DIR}/main-done - echo "==== Output ===========================================================================================" echo "| Code snippet ID: $CODE_SNIPPET_ID" echo "| Rootfs: $FINAL_FC_ROOTFS" From 4c168c7e10f9a8e1f9264ad30dbf0d7e40630d04 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Thu, 24 Aug 2023 17:04:39 +0200 Subject: [PATCH 3/7] Cleanup env vars handling in API --- packages/api/internal/api/key.go | 6 +----- packages/api/internal/handlers/store.go | 3 ++- packages/api/internal/nomad/client.go | 9 +++++++-- packages/api/internal/supabase/client.go | 7 +++++-- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/packages/api/internal/api/key.go b/packages/api/internal/api/key.go index 9b939e3529..5c7fb64d81 100644 --- a/packages/api/internal/api/key.go +++ b/packages/api/internal/api/key.go @@ -2,8 +2,4 @@ package api import "os" -const ( - APIAdminKeyName = "API_ADMIN_KEY" -) - -var APIAdminKey = os.Getenv(APIAdminKeyName) +var APIAdminKey = os.Getenv("API_ADMIN_KEY") diff --git a/packages/api/internal/handlers/store.go b/packages/api/internal/handlers/store.go index 3ed9b2043d..c5c7c19ad2 100644 --- a/packages/api/internal/handlers/store.go +++ b/packages/api/internal/handlers/store.go @@ -29,6 +29,8 @@ type APIStore struct { posthog posthog.Client } +var posthogAPIKey = os.Getenv("POSTHOG_API_KEY") + func NewAPIStore() *APIStore { fmt.Println("Initializing API store") @@ -68,7 +70,6 @@ func NewAPIStore() *APIStore { // Comment this line out if you are developing locally to prevent killing sessions in production go cache.KeepInSync(nomadClient) - posthogAPIKey := os.Getenv("POSTHOG_API_KEY") client, posthogErr := posthog.NewWithConfig(posthogAPIKey, posthog.Config{ Interval: 30 * time.Second, BatchSize: 100, diff --git a/packages/api/internal/nomad/client.go b/packages/api/internal/nomad/client.go index 3f055d7c80..d05413d8f8 100644 --- a/packages/api/internal/nomad/client.go +++ b/packages/api/internal/nomad/client.go @@ -10,14 +10,19 @@ import ( "github.com/hashicorp/nomad/api" ) +var ( + nomadAddress = os.Getenv("NOMAD_ADDRESS") + nomadToken = os.Getenv("NOMAD_TOKEN") +) + type NomadClient struct { client *api.Client } func InitNomadClient() *NomadClient { config := api.Config{ - Address: os.Getenv("NOMAD_ADDRESS"), - SecretID: os.Getenv("NOMAD_TOKEN"), + Address: nomadAddress, + SecretID: nomadToken, } client, err := api.NewClient(&config) diff --git a/packages/api/internal/supabase/client.go b/packages/api/internal/supabase/client.go index 0a1f79fcbf..38a9c36309 100644 --- a/packages/api/internal/supabase/client.go +++ b/packages/api/internal/supabase/client.go @@ -13,9 +13,12 @@ type DB struct { Client *postgrest.Client } +var ( + supabaseURL = os.Getenv("SUPABASE_URL") + supabaseKey = os.Getenv("SUPABASE_KEY") +) + func NewClient() (*DB, error) { - supabaseURL := os.Getenv("SUPABASE_URL") - supabaseKey := os.Getenv("SUPABASE_KEY") parsedURL, err := url.Parse(supabaseURL) if err != nil { From 3a7179c7031eb08134630ad3173736e942d8333c Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Mon, 28 Aug 2023 11:20:14 +0200 Subject: [PATCH 4/7] Restore part of the old openapi spec --- openapi.yml | 160 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 109 insertions(+), 51 deletions(-) diff --git a/openapi.yml b/openapi.yml index b3b2c49624..b3eae2914f 100644 --- a/openapi.yml +++ b/openapi.yml @@ -6,21 +6,21 @@ info: servers: - url: https://api.e2b.dev/ description: API endpoint - - url: https://{instanceID}-{clientID}.api.e2b.dev/ - description: Instance endpoint without a specified port + - url: https://{sessionID}-{clientID}.api.e2b.dev/ + description: Session endpoint without a specified port variables: - instanceID: - description: ID of the instance - default: _instanceID + sessionID: + description: ID of the session + default: _sessionID clientID: description: ID of the client default: _clientID - - url: https://{port}-{intanceID}-{clientID}.api.e2b.dev/ - description: Instance endpoint with a specificed port + - url: https://{port}-{sessionID}-{clientID}.api.e2b.dev/ + description: Session endpoint with a specificed port variables: - instanceID: - description: ID of the instance - default: _instanceID + sessionID: + description: ID of the session + default: _sessionID clientID: description: ID of the client default: _clientID @@ -33,23 +33,29 @@ components: ApiKeyAuth: name: api_key type: apiKey - in: header + in: query parameters: - apiKey: + apiKeyOpt: name: api_key - in: header + in: query + required: false + schema: + type: string + apiKeyReq: + name: api_key + in: query required: true schema: type: string - envID: - name: envID + codeSnippetID: + name: codeSnippetID in: path required: true schema: type: string - instanceID: - name: instanceID + sessionID: + name: sessionID in: path required: true schema: @@ -76,55 +82,85 @@ components: $ref: "#/components/schemas/Error" schemas: - EnvState: + Template: + type: string + enum: + - Nodejs + - Go + - Bash + - Rust + - Python3 + - PHP + - Java + - Perl + - DotNET + + EnvironmentState: type: string enum: - Building - Failed - Done - Env: + Environment: required: - id properties: id: type: string - name: + template: + type: string + title: type: string - NewEnv: + NewEnvironment: required: - - buildConfig + - template properties: - name: + title: type: string - buildConfig: + template: type: string - EnvStateUpdate: + EnvironmentStateUpdate: required: - state properties: state: - $ref: "#/components/schemas/EnvState" + $ref: "#/components/schemas/EnvironmentState" - EnvNameUpdate: + EnvironmentTitleUpdate: properties: - name: + title: type: string - Instance: + NewSession: + required: + - codeSnippetID + properties: + editEnabled: + type: boolean + default: false + description: Option determining if the session is a shared persistent edit session + codeSnippetID: + type: string + description: Identifier of a code snippet which which is the environment associated + Session: required: - - envID - - instanceID + - sessionID - clientID + - editEnabled + - codeSnippetID properties: - envID: + codeSnippetID: type: string - description: Identifier of the environment - instanceID: + description: Identifier of a code snippet which which is the environment associated + editEnabled: + type: boolean + description: Information if the session is a shared persistent edit session + sessionID: type: string - description: Identifier of the instance + description: Identifier of the session clientID: type: string description: Identifier of the client @@ -143,10 +179,10 @@ components: description: Error tags: - - name: instances - description: Managing environment instances + - name: sessions + description: Managing VM sessions - name: envs - description: Manage environments + description: Environment for VM paths: /health: @@ -163,7 +199,7 @@ paths: description: List all sessions tags: [sessions] parameters: - - $ref: "#/components/parameters/apiKey" + - $ref: "#/components/parameters/apiKeyReq" responses: 200: description: Successfully returned all sessions @@ -173,16 +209,16 @@ paths: type: array items: allOf: - - $ref: "#/components/schemas/Instance" + - $ref: "#/components/schemas/Session" 401: $ref: "#/components/responses/401" 500: $ref: "#/components/responses/500" post: - description: Create an instance of the environment + description: Create a session on the server tags: [sessions] parameters: - - $ref: "#/components/parameters/apiKey" + - $ref: "#/components/parameters/apiKeyOpt" requestBody: required: true content: @@ -242,7 +278,7 @@ paths: description: List all environments tags: [envs] parameters: - - $ref: "#/components/parameters/apiKey" + - $ref: "#/components/parameters/apiKeyReq" responses: 200: description: Successfully returned all environments @@ -261,7 +297,7 @@ paths: description: Create a new environment tags: [envs] parameters: - - $ref: "#/components/parameters/apiKey" + - $ref: "#/components/parameters/apiKeyReq" requestBody: required: true content: @@ -282,7 +318,29 @@ paths: 500: $ref: "#/components/responses/500" - /envs/{envID}: + # This method will be replaced by POST /envs + /envs/{codeSnippetID}: + post: + description: Create a new env for a code snippet + tags: [envs] + parameters: + - $ref: "#/components/parameters/apiKeyReq" + - $ref: "#/components/parameters/codeSnippetID" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/NewEnvironment" + responses: + 204: + description: Successfully created an environment + 400: + $ref: "#/components/responses/400" + 401: + $ref: "#/components/responses/401" + 500: + $ref: "#/components/responses/500" delete: description: Delete the code snippet environment tags: [envs] @@ -318,7 +376,7 @@ paths: 500: $ref: "#/components/responses/500" - /envs/{envID}/state: + /envs/{codeSnippetID}/state: put: description: Update the state of the environment tags: [envs] @@ -339,22 +397,22 @@ paths: 401: $ref: "#/components/responses/401" - /envs/{envID}/title: + /envs/{codeSnippetID}/title: put: description: Update the title of the environment tags: [envs] parameters: - - $ref: "#/components/parameters/apiKey" - - $ref: "#/components/parameters/envID" + - $ref: "#/components/parameters/apiKeyReq" + - $ref: "#/components/parameters/codeSnippetID" requestBody: required: true content: application/json: schema: - $ref: "#/components/schemas/EnvTitleUpdate" + $ref: "#/components/schemas/EnvironmentTitleUpdate" responses: 204: - description: Updated the title of the env + description: Updated the title of the code snippet 400: $ref: "#/components/responses/400" 401: From 009f3b9eacc43258038d763ef3616980fc76af14 Mon Sep 17 00:00:00 2001 From: Tomas Valenta Date: Tue, 29 Aug 2023 00:23:07 +0200 Subject: [PATCH 5/7] Update infra deployment [WIP] --- .terraform.lock.hcl | 38 +++++ main.tf | 162 +++++++++++----------- packages/api/Dockerfile | 1 - packages/cluster-disk-image/main.pkr.hcl | 2 +- packages/cluster/client/main.tf | 12 +- packages/cluster/server/main.tf | 24 +++- packages/firecracker-task-driver/Makefile | 2 +- variables.tf | 4 +- 8 files changed, 152 insertions(+), 93 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 96944be466..948f272812 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -1,6 +1,25 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. +provider "registry.terraform.io/hashicorp/consul" { + version = "2.18.0" + hashes = [ + "h1:GIbc9IimARunBuk+GdxgUX0cnZ2WwFW0IZMyT8E88B8=", + "zh:155f6899c8f1e0162169e4d09f426ea9a9738d74129f0ffc54b7570941c49cd4", + "zh:241ba909b387c349845b0371321af45ccbb00290edbe1ea0861ad8732951f718", + "zh:612f52be1fb5dd507b8064c67785313a531faad35873cba53f998f3766473335", + "zh:67fbd695381b9c5db83ffeeaec1bfdbcc477ccb1e9de1caff76a0186d4c85908", + "zh:7f88b151c9690a6addccbffe8484f0257344ef55424a9efb025dfddd052a4dc6", + "zh:8d954f3ffeb72b6c18cc5ae8c3189bb3a8cb66967b2106c7d0163009c12bba15", + "zh:913774c7eabc6e9078a1bd00347cc539b19a6f6b45dacbd21454dffdc9f4ae43", + "zh:9517558883f994649695643a6208079ed0445aaa0ac2dee69f88cb044d21c6c9", + "zh:a0211f596e35bd1b8d4bb9cda321cb1555a427a8d3f6724fe09893168fac9b7e", + "zh:a70eaa0a88f677f901855a4ab908ddcf961e4afaf3a0147b08faaead57b4fe07", + "zh:b02f5cd94ab236d988cfec531c56c199e3087803f1b2908a1a2b6da8a57b3751", + "zh:f3d3efac504c9484a025beb919d22b290aa6dbff256f6e86c1f8ce7817e077e5", + ] +} + provider "registry.terraform.io/hashicorp/google" { version = "3.90.1" constraints = "~> 3.53" @@ -43,6 +62,25 @@ provider "registry.terraform.io/hashicorp/google-beta" { ] } +provider "registry.terraform.io/hashicorp/nomad" { + version = "1.4.20" + hashes = [ + "h1:4IKRa9RKAwJ55lLQjgkOkXLnm+KV4T0bCH+ldNhRR14=", + "zh:02989edcebe724fc0aa873b22176fd20074c4f46295e728010711a8fc5dfa72c", + "zh:089ba7d19bcf5c6bab3f8b8c5920eb6d78c52cf79bb0c5dfeb411c600e7efcba", + "zh:235865a2182ca372bcbf440201a8b8cc0715ad5dbc4de893d99b6f32b5be53ab", + "zh:67ea718764f3f344ecc6e027d20c1327b86353c8064aa90da3ec12cec4a88954", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:8c68c540f0df4980568bdd688c2adec86eda62eb2de154e3db215b16de0a7ae0", + "zh:911969c63a69a733be57b96d54c5966c9424e1abec8d5f20038c8cef3a504c65", + "zh:a673c92ddc9d47e8d53dcb9b376f1adcb4543488202fc83a3e7eab8677530684", + "zh:a94a73eae89fd8c8ebf872013079be41161d3f293f4026c92d45c4c5667dd613", + "zh:db6b89f8b696040c0344f00928e4cf6e0a75034421ba14cdcd8a4d23bc865dce", + "zh:e512c0b1239e3d66b60d22c2b4de19fea288e492cde90dff9277cc475fd9dbbf", + "zh:ef6eccecbdef3bb8ce629cabfb5550c1db5c3e952943dda1786ef6cb470a8c23", + ] +} + provider "registry.terraform.io/kreuzwerker/docker" { version = "2.16.0" constraints = "2.16.0" diff --git a/main.tf b/main.tf index 60ad5194e6..6e046bf8e5 100644 --- a/main.tf +++ b/main.tf @@ -57,84 +57,84 @@ data "google_secret_manager_secret_version" "consul_acl_token" { secret = "consul-secret-id" } -# provider "consul" { -# address = "http://${data.google_compute_global_address.orch_server_consul_ip.address}" -# token = data.google_secret_manager_secret_version.consul_acl_token.secret_data -# } - -# resource "consul_acl_policy" "agent" { -# name = "agent" -# rules = <<-RULE -# key_prefix "" { -# policy = "deny" -# } -# RULE -# } - -# resource "consul_acl_token_policy_attachment" "attachment" { -# token_id = "00000000-0000-0000-0000-000000000002" -# policy = consul_acl_policy.agent.name -# } - -# data "google_compute_global_address" "orch_server_ip" { -# name = "orch-server-ip" -# } - -# data "google_secret_manager_secret_version" "nomad_acl_token" { -# secret = "nomad-secret-id" -# } - -# provider "nomad" { -# address = "http://${data.google_compute_global_address.orch_server_ip.address}" -# secret_id = data.google_secret_manager_secret_version.nomad_acl_token.secret_data -# } - -# data "google_secret_manager_secret_version" "lightstep_api_key" { -# secret = "lightstep-api-key" -# } - -# data "google_secret_manager_secret_version" "betterstack_logs_api_key" { -# secret = "betterstack-logs-api-key" -# } - -# module "telemetry" { -# source = "./packages/telemetry" - -# logs_health_proxy_port = var.logs_health_proxy_port -# logs_proxy_port = var.logs_proxy_port -# lightstep_api_key = data.google_secret_manager_secret_version.lightstep_api_key.secret_data -# betterstack_logs_api_key = data.google_secret_manager_secret_version.betterstack_logs_api_key.secret_data -# gcp_zone = var.gcp_zone -# } - -# module "session_proxy" { -# source = "./packages/session-proxy" - -# client_cluster_size = var.client_cluster_size -# gcp_zone = var.gcp_zone -# session_proxy_service_name = var.session_proxy_service_name - -# session_proxy_port = var.session_proxy_port -# } - -# module "client_proxy" { -# source = "./packages/client-proxy" - -# gcp_zone = var.gcp_zone -# session_proxy_service_name = var.session_proxy_service_name - -# client_proxy_port = var.client_proxy_port -# client_proxy_health_port = var.client_proxy_health_port -# } - -# module "api" { -# source = "./packages/api" - -# gcp_zone = var.gcp_zone - -# logs_proxy_address = "http://${module.cluster.logs_proxy_ip}" -# nomad_address = "http://${module.cluster.server_proxy_ip}" -# nomad_token = data.google_secret_manager_secret_version.nomad_acl_token.secret_data -# consul_token = data.google_secret_manager_secret_version.consul_acl_token.secret_data -# api_port = var.api_port -# } +provider "consul" { + address = "http://${data.google_compute_global_address.orch_server_consul_ip.address}" + token = data.google_secret_manager_secret_version.consul_acl_token.secret_data +} + +resource "consul_acl_policy" "agent" { + name = "agent" + rules = <<-RULE + key_prefix "" { + policy = "deny" + } + RULE +} + +resource "consul_acl_token_policy_attachment" "attachment" { + token_id = "00000000-0000-0000-0000-000000000002" + policy = consul_acl_policy.agent.name +} + +data "google_compute_global_address" "orch_server_ip" { + name = "orch-server-nomad-ip" +} + +data "google_secret_manager_secret_version" "nomad_acl_token" { + secret = "nomad-secret-id" +} + +provider "nomad" { + address = "http://${data.google_compute_global_address.orch_server_ip.address}" + secret_id = data.google_secret_manager_secret_version.nomad_acl_token.secret_data +} + +data "google_secret_manager_secret_version" "lightstep_api_key" { + secret = "lightstep-api-key" +} + +data "google_secret_manager_secret_version" "betterstack_logs_api_key" { + secret = "betterstack-logs-api-key" +} + +module "telemetry" { + source = "./packages/telemetry" + + logs_health_proxy_port = var.logs_health_proxy_port + logs_proxy_port = var.logs_proxy_port + lightstep_api_key = data.google_secret_manager_secret_version.lightstep_api_key.secret_data + betterstack_logs_api_key = data.google_secret_manager_secret_version.betterstack_logs_api_key.secret_data + gcp_zone = var.gcp_zone +} + +module "session_proxy" { + source = "./packages/session-proxy" + + client_cluster_size = var.client_cluster_size + gcp_zone = var.gcp_zone + session_proxy_service_name = var.session_proxy_service_name + + session_proxy_port = var.session_proxy_port +} + +module "client_proxy" { + source = "./packages/client-proxy" + + gcp_zone = var.gcp_zone + session_proxy_service_name = var.session_proxy_service_name + + client_proxy_port = var.client_proxy_port + client_proxy_health_port = var.client_proxy_health_port +} + +module "api" { + source = "./packages/api" + + gcp_zone = var.gcp_zone + + logs_proxy_address = "http://${module.cluster.logs_proxy_ip}" + nomad_address = "http://${module.cluster.server_proxy_ip}" + nomad_token = data.google_secret_manager_secret_version.nomad_acl_token.secret_data + consul_token = data.google_secret_manager_secret_version.consul_acl_token.secret_data + api_port = var.api_port +} diff --git a/packages/api/Dockerfile b/packages/api/Dockerfile index 473246be64..893d574778 100644 --- a/packages/api/Dockerfile +++ b/packages/api/Dockerfile @@ -22,4 +22,3 @@ RUN chmod +x api # Set Gin server to the production mode ENV GIN_MODE=release ENTRYPOINT [ "./api"] - \ No newline at end of file diff --git a/packages/cluster-disk-image/main.pkr.hcl b/packages/cluster-disk-image/main.pkr.hcl index 64a3b7f20a..d1a79a046e 100644 --- a/packages/cluster-disk-image/main.pkr.hcl +++ b/packages/cluster-disk-image/main.pkr.hcl @@ -14,7 +14,7 @@ source "googlecompute" "orch" { # TODO: Overwrite the image instead of creating timestamped images every time we build its image_name = "orch-${formatdate("YYYY-MM-DD-hh-mm-ss", timestamp())}" project_id = var.gcp_project_id - source_image_family = "ubuntu-2204-lts" + source_image_family = "ubuntu-2004-lts" ssh_username = "ubuntu" zone = var.gcp_zone disk_size = 10 diff --git a/packages/cluster/client/main.tf b/packages/cluster/client/main.tf index 4dae3aebf6..06aba4bea6 100644 --- a/packages/cluster/client/main.tf +++ b/packages/cluster/client/main.tf @@ -126,7 +126,7 @@ resource "google_compute_instance_template" "client" { # This cert is for proxying throught Cloudflare only data "google_compute_ssl_certificate" "session_certificate" { - name = "sessions" + name = "e2b-sessions" } # This should be SSL cert for usage withotu Cloudflare @@ -139,12 +139,12 @@ resource "google_compute_url_map" "client_map" { default_service = module.gce_lb_http.backend_services["session"].self_link host_rule { - hosts = ["api.e2b.com"] + hosts = ["api.e2b.dev"] path_matcher = "api-paths" } host_rule { - hosts = ["*.api.e2b.com"] + hosts = ["*.e2b.dev"] path_matcher = "session-paths" } @@ -159,8 +159,8 @@ resource "google_compute_url_map" "client_map" { } } -data "google_compute_global_address" "orch_client_ip" { - name = "orch-client-ip" +data "google_compute_global_address" "orch_client_api_ip" { + name = "orch-client-api-ip" } module "gce_lb_http" { @@ -168,7 +168,7 @@ module "gce_lb_http" { version = "~> 5.1" name = "orch-external-session" project = var.gcp_project_id - address = data.google_compute_global_address.orch_client_ip.address + address = data.google_compute_global_address.orch_client_api_ip.address ssl_certificates = [ data.google_compute_ssl_certificate.session_certificate.self_link, data.google_compute_ssl_certificate.api_certificate.self_link, diff --git a/packages/cluster/server/main.tf b/packages/cluster/server/main.tf index b341e16c07..46d9ba702b 100644 --- a/packages/cluster/server/main.tf +++ b/packages/cluster/server/main.tf @@ -116,7 +116,11 @@ resource "google_compute_instance_template" "server" { # LOAD BALANCERS data "google_compute_global_address" "orch_server_ip" { - name = "orch-server-ip" + name = "orch-server-nomad-ip" +} + +data "google_compute_ssl_certificate" "nomad_certificate" { + name = "e2b-nomad-api" } module "gce_lb_http_nomad" { @@ -129,6 +133,13 @@ module "gce_lb_http_nomad" { target_tags = [ var.cluster_tag_name, ] + ssl_certificates = [ + data.google_compute_ssl_certificate.nomad_certificate.self_link, + ] + use_ssl_certificates = true + ssl = true + + firewall_networks = [var.network_name] backends = { @@ -191,6 +202,10 @@ data "google_compute_global_address" "orch_server_consul_ip" { name = "orch-server-consul-ip" } +data "google_compute_ssl_certificate" "consul_certificate" { + name = "e2b-consul-api" +} + module "gce_lb_http_consul" { source = "GoogleCloudPlatform/lb-http/google" version = "~> 5.1" @@ -201,6 +216,13 @@ module "gce_lb_http_consul" { target_tags = [ var.cluster_tag_name, ] + + ssl_certificates = [ + data.google_compute_ssl_certificate.consul_certificate.self_link, + ] + use_ssl_certificates = true + ssl = true + firewall_networks = [var.network_name] backends = { diff --git a/packages/firecracker-task-driver/Makefile b/packages/firecracker-task-driver/Makefile index 60a3ebf22c..fb09b1eb39 100644 --- a/packages/firecracker-task-driver/Makefile +++ b/packages/firecracker-task-driver/Makefile @@ -7,7 +7,7 @@ build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/ . set-project-gcloud: - gcloud config set project devbookhq + gcloud config set project e2b-prod generate: cd internal/client && swagger generate client -f firecracker.yml -A firecracker diff --git a/variables.tf b/variables.tf index 791077b02b..d789e91bbe 100644 --- a/variables.tf +++ b/variables.tf @@ -22,7 +22,7 @@ variable "server_cluster_size" { variable "server_machine_type" { type = string - default = "n1-standard-4" + default = "n1-standard-1" } variable "client_cluster_size" { @@ -32,7 +32,7 @@ variable "client_cluster_size" { variable "client_machine_type" { type = string - default = "n1-standard-8" + default = "n1-standard-1" } variable "client_proxy_health_port" { From 7c93def1f5d5c1981a06994fab4482e25387b9ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Nov=C3=A1k?= Date: Wed, 30 Aug 2023 17:17:49 +0200 Subject: [PATCH 6/7] Fix analytics for closing the session (#3) --- packages/api/internal/handlers/sessions.go | 5 +- packages/api/internal/handlers/store.go | 39 ++++++---- packages/api/internal/nomad/cache.go | 83 +++++++++------------- packages/api/internal/nomad/session.go | 12 ---- 4 files changed, 61 insertions(+), 78 deletions(-) diff --git a/packages/api/internal/handlers/sessions.go b/packages/api/internal/handlers/sessions.go index b27f7d1ea5..b0fcf17433 100644 --- a/packages/api/internal/handlers/sessions.go +++ b/packages/api/internal/handlers/sessions.go @@ -3,6 +3,7 @@ package handlers import ( "fmt" "net/http" + "time" "github.com/devbookhq/devbook-api/packages/api/internal/api" "github.com/gin-gonic/gin" @@ -122,6 +123,7 @@ func (a *APIStore) PostSessions( a.sendAPIStoreError(c, err.Code, err.ClientMsg) return } + startTime := time.Now() ReportEvent(ctx, "created session") if teamID != nil { err := a.posthog.Enqueue(posthog.GroupIdentify{ @@ -148,7 +150,6 @@ func (a *APIStore) PostSessions( if err != nil { fmt.Printf("Error when sending event to Posthog: %+v\n", err) } - a.teamCache.Add(session.SessionID, *teamID) } if *newSession.EditEnabled { @@ -175,7 +176,7 @@ func (a *APIStore) PostSessions( } } - if err := a.sessionsCache.Add(session); err != nil { + if err := a.sessionsCache.Add(session, teamID, &startTime); err != nil { errMsg := fmt.Errorf("error when adding session to cache: %v", err) ReportError(ctx, errMsg) diff --git a/packages/api/internal/handlers/store.go b/packages/api/internal/handlers/store.go index 3ed9b2043d..fe7a25d4b4 100644 --- a/packages/api/internal/handlers/store.go +++ b/packages/api/internal/handlers/store.go @@ -19,7 +19,6 @@ import ( type APIStore struct { sessionsCache *nomad.SessionCache - teamCache *nomad.TeamCache nomad *nomad.NomadClient supabase *supabase.DB NextId int64 @@ -63,13 +62,8 @@ func NewAPIStore() *APIStore { fmt.Fprintf(os.Stderr, "Error loading current sessions from Nomad\n: %s", sessionErr) } - cache := nomad.NewSessionCache(nomadClient.DeleteSession, initialSessions) - teamCache := nomad.NewTeamCache() - // Comment this line out if you are developing locally to prevent killing sessions in production - go cache.KeepInSync(nomadClient) - posthogAPIKey := os.Getenv("POSTHOG_API_KEY") - client, posthogErr := posthog.NewWithConfig(posthogAPIKey, posthog.Config{ + posthogClient, posthogErr := posthog.NewWithConfig(posthogAPIKey, posthog.Config{ Interval: 30 * time.Second, BatchSize: 100, Verbose: true, @@ -80,15 +74,18 @@ func NewAPIStore() *APIStore { panic(posthogErr) } + cache := nomad.NewSessionCache(getDeleteSessionFunction(nomadClient, posthogClient), initialSessions) + // Comment this line out if you are developing locally to prevent killing sessions in production + go cache.KeepInSync(nomadClient) + return &APIStore{ nomad: nomadClient, supabase: supabaseClient, NextId: 1000, sessionsCache: cache, - teamCache: teamCache, tracer: tracer, templates: templates, - posthog: client, + posthog: posthogClient, } } @@ -178,7 +175,19 @@ func (a *APIStore) validateTeamAPIKey(apiKey *string) *string { } func (a *APIStore) DeleteSession(sessionID string, purge bool) *api.APIError { - duration, err := a.nomad.DeleteSessionWithDuration(sessionID, purge) + sessionData := a.sessionsCache.Get(sessionID) + return deleteSession(a.nomad, a.posthog, sessionID, sessionData.TeamID, sessionData.StartTime, purge) +} + +type sessionData = nomad.SessionData + +func getDeleteSessionFunction(nomad *nomad.NomadClient, posthogClient posthog.Client) func(sessionData sessionData, purge bool) *api.APIError { + return func(sessionData sessionData, purge bool) *api.APIError { + return deleteSession(nomad, posthogClient, sessionData.Session.SessionID, sessionData.TeamID, sessionData.StartTime, purge) + } +} +func deleteSession(nomad *nomad.NomadClient, posthogClient posthog.Client, sessionID string, teamID *string, startTime *time.Time, purge bool) *api.APIError { + err := nomad.DeleteSession(sessionID, purge) if err != nil { return &api.APIError{ Msg: fmt.Sprintf("cannot delete session '%s': %+v", sessionID, err), @@ -186,13 +195,13 @@ func (a *APIStore) DeleteSession(sessionID string, purge bool) *api.APIError { Code: http.StatusInternalServerError, } } - teamID, teamErr := a.teamCache.Get(sessionID) - if teamErr != nil { - err := a.posthog.Enqueue(posthog.Capture{ - DistinctId: "backend_infra", + + if teamID != nil && startTime != nil { + err := posthogClient.Enqueue(posthog.Capture{ + DistinctId: "backend", Event: "closed_session", Properties: posthog.NewProperties(). - Set("session_id", sessionID).Set("duration", duration), + Set("session_id", sessionID).Set("duration", time.Since(*startTime).Seconds()), Groups: posthog.NewGroups(). Set("team", teamID), }) diff --git a/packages/api/internal/nomad/cache.go b/packages/api/internal/nomad/cache.go index 3c7f45a14b..e245c312d8 100644 --- a/packages/api/internal/nomad/cache.go +++ b/packages/api/internal/nomad/cache.go @@ -15,17 +15,27 @@ const ( cacheSyncTime = time.Second * 180 ) +type SessionData struct { + Session *api.Session + TeamID *string + StartTime *time.Time +} + type SessionCache struct { - cache *ttlcache.Cache[string, *api.Session] + cache *ttlcache.Cache[string, SessionData] } // Add the session to the cache and start expiration timer -func (c *SessionCache) Add(session *api.Session) error { +func (c *SessionCache) Add(session *api.Session, teamID *string, startTime *time.Time) error { if c.Exists(session.SessionID) { return fmt.Errorf("session \"%s\" already exists", session.SessionID) } - - c.cache.Set(session.SessionID, session, ttlcache.DefaultTTL) + sessionData := SessionData{ + Session: session, + TeamID: teamID, + StartTime: startTime, + } + c.cache.Set(session.SessionID, sessionData, ttlcache.DefaultTTL) return nil } @@ -39,21 +49,31 @@ func (c *SessionCache) Refresh(sessionID string) error { return nil } +// Get the session from the cache +func (c *SessionCache) Get(sessionID string) SessionData { + item := c.cache.Get(sessionID, ttlcache.WithDisableTouchOnHit[string, SessionData]()) + if item != nil { + return item.Value() + } else { + panic(fmt.Errorf("session \"%s\" doesn't exist", sessionID)) + } +} + // Check if the session exists in the cache func (c *SessionCache) Exists(sessionID string) bool { - item := c.cache.Get(sessionID, ttlcache.WithDisableTouchOnHit[string, *api.Session]()) + item := c.cache.Get(sessionID, ttlcache.WithDisableTouchOnHit[string, SessionData]()) return item != nil } func (c *SessionCache) FindEditSession(codeSnippetID string) (*api.Session, error) { for _, item := range c.cache.Items() { - if item.Value() == nil { + if item.Value().Session == nil { continue } - if item.Value().EditEnabled && item.Value().CodeSnippetID == codeSnippetID { + if item.Value().Session.EditEnabled && item.Value().Session.CodeSnippetID == codeSnippetID { c.Refresh(item.Key()) - return item.Value(), nil + return item.Value().Session, nil } } return nil, fmt.Errorf("error edit session for code snippet '%s' not found", codeSnippetID) @@ -62,21 +82,21 @@ func (c *SessionCache) FindEditSession(codeSnippetID string) (*api.Session, erro func (c *SessionCache) Sync(sessions []*api.Session) { for _, session := range sessions { if !c.Exists(session.SessionID) { - c.Add(session) + c.Add(session, nil, nil) } } } // We will need to either use Redis for storing active sessions OR retrieve them from Nomad when we start API to keep everything in sync // We are retrieving the tasks from Nomad now -func NewSessionCache(handleDeleteSession func(sessionID string, purge bool) *api.APIError, initialSessions []*api.Session) *SessionCache { +func NewSessionCache(handleDeleteSession func(sessionData SessionData, purge bool) *api.APIError, initialSessions []*api.Session) *SessionCache { cache := ttlcache.New( - ttlcache.WithTTL[string, *api.Session](sessionExpiration), + ttlcache.WithTTL[string, SessionData](sessionExpiration), ) - cache.OnEviction(func(ctx context.Context, er ttlcache.EvictionReason, i *ttlcache.Item[string, *api.Session]) { + cache.OnEviction(func(ctx context.Context, er ttlcache.EvictionReason, i *ttlcache.Item[string, SessionData]) { if er == ttlcache.EvictionReasonExpired || er == ttlcache.EvictionReasonDeleted { - err := handleDeleteSession(i.Key(), true) + err := handleDeleteSession(i.Value(), true) if err != nil { fmt.Fprintf(os.Stderr, "Error deleting session (%v)\n: %s", er, err.Error()) } @@ -88,7 +108,7 @@ func NewSessionCache(handleDeleteSession func(sessionID string, purge bool) *api } for _, session := range initialSessions { - sessionCache.Add(session) + sessionCache.Add(session, nil, nil) } go cache.Start() @@ -112,38 +132,3 @@ func (c *SessionCache) KeepInSync(client *NomadClient) { func (c *SessionCache) Count() int { return c.cache.Len() } - -type TeamCache struct { - cache *ttlcache.Cache[string, string] -} - -func (c *TeamCache) Add(sessionID string, teamID string) { - c.cache.Set(sessionID, teamID, ttlcache.DefaultTTL) -} - -// Check if the session exists in the cache -func (c *TeamCache) Exists(sessionID string) bool { - item := c.cache.Get(sessionID, ttlcache.WithDisableTouchOnHit[string, string]()) - return item != nil -} - -func (c *TeamCache) Get(sessionID string) (string, error) { - item := c.cache.Get(sessionID) - if item == nil { - return "", fmt.Errorf("no team for the session \"%s\" found", sessionID) - - } - return item.Value(), nil -} -func NewTeamCache() *TeamCache { - cache := ttlcache.New( - ttlcache.WithTTL[string, string](sessionExpiration), - ) - - teamCache := &TeamCache{ - cache: cache, - } - go cache.Start() - - return teamCache -} diff --git a/packages/api/internal/nomad/session.go b/packages/api/internal/nomad/session.go index 40dff09451..5e016beb62 100644 --- a/packages/api/internal/nomad/session.go +++ b/packages/api/internal/nomad/session.go @@ -188,15 +188,3 @@ func (n *NomadClient) DeleteSession(sessionID string, purge bool) *api.APIError } return nil } - -func (n *NomadClient) DeleteSessionWithDuration(sessionID string, purge bool) (*time.Duration, *api.APIError) { - _, meta, err := n.client.Jobs().Deregister(sessionsJobNameWithSlash+sessionID, purge, &nomadAPI.WriteOptions{}) - if err != nil { - return nil, &api.APIError{ - Msg: fmt.Sprintf("cannot delete job '%s%s' job: %+v", sessionsJobNameWithSlash, sessionID, err), - ClientMsg: "Cannot delete the session right now", - Code: http.StatusInternalServerError, - } - } - return &meta.RequestTime, nil -} From b06d38cda938fd4a66c7d4ee886ddfed214be9cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Nov=C3=A1k?= Date: Thu, 31 Aug 2023 11:56:25 +0200 Subject: [PATCH 7/7] Require API key (#4) * Require API key * Improve missing api key link * Add smol developer * Increase version --- VERSION | 2 +- packages/api/internal/handlers/sessions.go | 7 ++++--- packages/cli/package.json | 2 +- packages/js-sdk/package-lock.json | 4 ++-- packages/js-sdk/package.json | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/VERSION b/VERSION index f49db97971..5156192cc4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.6.88 \ No newline at end of file +2.6.89 \ No newline at end of file diff --git a/packages/api/internal/handlers/sessions.go b/packages/api/internal/handlers/sessions.go index b0fcf17433..527678184e 100644 --- a/packages/api/internal/handlers/sessions.go +++ b/packages/api/internal/handlers/sessions.go @@ -103,14 +103,15 @@ func (a *APIStore) PostSessions( } var teamID *string - if a.isPredefinedTemplate(newSession.CodeSnippetID) { + // PPSrlH5TIvFx is smol development environment id + if a.isPredefinedTemplate(newSession.CodeSnippetID) || newSession.CodeSnippetID == "PPSrlH5TIvFx" { teamID = a.validateTeamAPIKey(params.ApiKey) - if teamID == nil && params.ApiKey != nil { + if teamID == nil { _, _, userErr := a.validateAPIKey(params.ApiKey) if userErr != nil { errMsg := fmt.Errorf("invalid API key: %+v", params.ApiKey) ReportCriticalError(ctx, errMsg) - a.sendAPIStoreError(c, 401, "Invalid API key") + a.sendAPIStoreError(c, 401, "Invalid API key, please visit https://e2b.dev/docs?reason=sdk-missing-api-key to get your API key.") return } } diff --git a/packages/cli/package.json b/packages/cli/package.json index fdc1c3361b..2f3f7fa5e9 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@devbookhq/cli", - "version": "2.6.88", + "version": "2.6.89", "description": "CLI for managing Devbook environments", "homepage": "https://usedevbook.com", "license": "SEE LICENSE IN LICENSE", diff --git a/packages/js-sdk/package-lock.json b/packages/js-sdk/package-lock.json index e147396d20..8a2acbe456 100644 --- a/packages/js-sdk/package-lock.json +++ b/packages/js-sdk/package-lock.json @@ -1,12 +1,12 @@ { "name": "@devbookhq/sdk", - "version": "2.6.88", + "version": "2.6.89", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@devbookhq/sdk", - "version": "2.6.88", + "version": "2.6.89", "license": "SEE LICENSE IN LICENSE", "dependencies": { "cross-fetch": "^3.1.5", diff --git a/packages/js-sdk/package.json b/packages/js-sdk/package.json index 10c68358bb..b2f225c297 100644 --- a/packages/js-sdk/package.json +++ b/packages/js-sdk/package.json @@ -1,6 +1,6 @@ { "name": "@devbookhq/sdk", - "version": "2.6.88", + "version": "2.6.89", "description": "SDK for managing Devbook sessions from JavaScript/TypeScript", "homepage": "https://usedevbook.com", "license": "SEE LICENSE IN LICENSE",