Deploy infra in a new project

.github/disabled-workflows/devbookd.yml

@@ -42,7 +42,7 @@ jobs:
           service_account: ${{ secrets.service_account_email }}
 
       - name: Upload devbookd
-        uses: 'google-github-actions/upload-cloud-storage@v1'
+        uses: "google-github-actions/upload-cloud-storage@v1"
         with:
-          path: './packages/devbookd/dist/devbookd_linux_amd64_v1/devbookd'
-          destination: 'devbook-environment-pipeline'
+          path: "./packages/devbookd/dist/devbookd_linux_amd64_v1/devbookd"
+          destination: "e2b-fc-env-pipeline"

.github/disabled-workflows/fc-env.yml

@@ -27,7 +27,7 @@ jobs:
           service_account: ${{ secrets.service_account_email }}
 
       - name: Upload env pipelines
-        uses: 'google-github-actions/upload-cloud-storage@v1'
+        uses: "google-github-actions/upload-cloud-storage@v1"
         with:
-          path: './packages/fc-env/env.tar.gz'
-          destination: 'devbook-environment-pipeline'
+          path: "./packages/fc-env/env.tar.gz"
+          destination: "e2b-fc-env-pipeline"

.github/disabled-workflows/firecracker-task-driver.yml

@@ -16,14 +16,13 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
 
-
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
           go-version-file: ./packages/firecracker-task-driver/go.mod
           cache: true
           cache-dependency-path: ./packages/firecracker-task-driver/go.sum
-          
+
       - name: Download deps
         working-directory: ./packages/firecracker-task-driver
         run: go mod tidy
@@ -39,7 +38,7 @@ jobs:
           service_account: ${{ secrets.service_account_email }}
 
       - name: Upload firecracker task driver
-        uses: 'google-github-actions/upload-cloud-storage@v1'
+        uses: "google-github-actions/upload-cloud-storage@v1"
         with:
-          path: './packages/firecracker-task-driver/bin/firecracker-task-driver'
-          destination: 'devbook-environment-pipeline'
+          path: "./packages/firecracker-task-driver/bin/firecracker-task-driver"
+          destination: "e2b-fc-env-pipeline"

.gitignore

@@ -3,3 +3,5 @@ core.3071
 node_modules
 # Ignore Vale.sh styles directory
 vale-styles
+
+.env

DEV.md

@@ -2,8 +2,6 @@
 
 ## Infra
 
-- Clock drift
-  - https://github.com/firecracker-microvm/firecracker/blob/eb8de3ba1f7cb636d2aaa632fe96b234f3a302e6/FAQ.md#my-guest-wall-clock-is-drifting-how-can-i-fix-it
 - Change read/write file to allow other thatn utf-8 format so we don't break the files
 - Remove dnsmasq from cluster VM image if it is not needed
 - Remove need for provisioning during building env
@@ -23,8 +21,8 @@
 - Check if the kernel args we are using are what we want
 - Improve API (efficient calls Nomad calls, no polling, etc.)
 - Improve FC WS connection (subscriptions take additional calls, maybe we can improve that, etc.)
-- Move to ubuntu
 - Update kernel version
+- devbookd jsonrpc parameters could be objects instead of arrays (compatibility advantages)
 - Move to a better FS handling (filestore, fuse?)
 - Fix possible mutex problems in the firecracker task driver
 - Update devbookd in all envs automatically
@@ -53,17 +51,14 @@
 - Use the FC jailer properly
 - Rebuild only the changed templates on push
 - Update NodeJS version
-- Remove top package-lock
 - Remove gitpod deps and mentions (makefile, gitpod yaml)
 - Remove devbook specific code from the repo (allow move to new org or on prem deploy)
 - Improve caching of the GH actions
 - Add instance sizes and other things as variables in the GH actions and remove them from codes
 - API URL should be configurable
 - add testing/staging env
-- create proper monorepo for the packages (pnpm for Node) and improve release system
 - Add memory swap to Rust - OoM when compiling bigger programs
   - https://www.digitalocean.com/community/tutorials/how-to-add-swap-space-on-ubuntu-22-04
-- Should we use ubuntu 22 for envs?
 - Does ubuntu need the newer kernel?
   - Using driver with multiple kernels
 - Cloud Hypervisor insted of FC - https://github.com/cloud-hypervisor/cloud-hypervisor
@@ -127,6 +122,7 @@
 - Add sync support to the python SDK and js sdk
   - the ws pong is working now but the actuall message backpressure can still make problems
 - Add golangci-lint
+- use dnsmasq instead of /etc/hosts and second proxy
 
 
 ### Python SDK
@@ -145,7 +141,7 @@
 - add tests
 - support lower python versions
 - Check if stderr and stdout are ordered
-- Myabe remove types (and timestamps?) from stdout/stderr
+- Myabe remove types (and timestamps?) from stdout/stderr (they are already fully identified by the subscription)
 - Use specific version (nodejs20) for the templates
 - Using pathlib for paths? windows support?
 - Flush all stdout/err after killing process or terminal in devbookd and also wait for the Stdout/err in the SDK

ENVS.md

@@ -0,0 +1,8 @@
+# Custom Envs
+
+- How to create a custom env from dockerfile if you want to use local files during the build step?
+  - It's either you build the dockerfile locally or you package and upload the files to our infra so it can be used during the build step.
+
+
+
+- Can we has the dockerfile+files+scripts/config to get the unique env hash that we can use to start identical envs in the future?

Makefile

@@ -7,7 +7,7 @@ login-gcloud:
 .PHONY: login-gcloud-user
 login-gcloud-user:
 	gcloud auth login
-	gcloud config set project devbookhq
+	gcloud config set project e2b-prod
 	gcloud --quiet auth configure-docker us-central1-docker.pkg.dev
 
 .PHONY: init

main.tf

@@ -1,7 +1,7 @@
 terraform {
   required_version = ">=1.1.9"
   backend "gcs" {
-    bucket = "devbook-terraform-state"
+    bucket = "e2b-terraform-state"
     prefix = "terraform/orchestration/state"
   }
   required_providers {
@@ -77,7 +77,7 @@ resource "consul_acl_token_policy_attachment" "attachment" {
 }
 
 data "google_compute_global_address" "orch_server_ip" {
-  name = "orch-server-ip"
+  name = "orch-server-nomad-ip"
 }
 
 data "google_secret_manager_secret_version" "nomad_acl_token" {
@@ -93,18 +93,18 @@ data "google_secret_manager_secret_version" "lightstep_api_key" {
   secret = "lightstep-api-key"
 }
 
-data "google_secret_manager_secret_version" "logtail_api_key" {
-  secret = "logtail-api-key"
+data "google_secret_manager_secret_version" "betterstack_logs_api_key" {
+  secret = "betterstack-logs-api-key"
 }
 
 module "telemetry" {
   source = "./packages/telemetry"
 
-  logs_health_proxy_port = var.logs_health_proxy_port
-  logs_proxy_port        = var.logs_proxy_port
-  lightstep_api_key      = data.google_secret_manager_secret_version.lightstep_api_key.secret_data
-  logtail_api_key        = data.google_secret_manager_secret_version.logtail_api_key.secret_data
-  gcp_zone               = var.gcp_zone
+  logs_health_proxy_port   = var.logs_health_proxy_port
+  logs_proxy_port          = var.logs_proxy_port
+  lightstep_api_key        = data.google_secret_manager_secret_version.lightstep_api_key.secret_data
+  betterstack_logs_api_key = data.google_secret_manager_secret_version.betterstack_logs_api_key.secret_data
+  gcp_zone                 = var.gcp_zone
 }
 
 module "session_proxy" {

openapi.yml

@@ -1,13 +1,12 @@
 openapi: 3.0.0
 info:
-  version: 1.0.0
-  title: Devbook
-  description: Devbook API
+  version: 0.1.0
+  title: E2B API
 
 servers:
-  - url: https://ondevbook.com/
+  - url: https://api.e2b.dev/
     description: API endpoint
-  - url: https://{sessionID}-{clientID}.ondevbook.com/
+  - url: https://{sessionID}-{clientID}.api.e2b.dev/
     description: Session endpoint without a specified port
     variables:
       sessionID:
@@ -16,7 +15,7 @@ servers:
       clientID:
         description: ID of the client
         default: _clientID
-  - url: https://{port}-{sessionID}-{clientID}.ondevbook.com/
+  - url: https://{port}-{sessionID}-{clientID}.api.e2b.dev/
     description: Session endpoint with a specificed port
     variables:
       sessionID:

packages/api/Dockerfile

@@ -22,4 +22,3 @@ RUN chmod +x api
 # Set Gin server to the production mode
 ENV GIN_MODE=release
 ENTRYPOINT [ "./api"]
-

packages/api/Makefile

@@ -1,6 +1,6 @@
 openapi := ../../openapi.yml
 
-GCP_PROJECT := devbookhq
+GCP_PROJECT := e2b-prod
 IMAGE := orchestration/api
 
 generate:
@@ -9,7 +9,7 @@ generate:
 	oapi-codegen -old-config-style -generate spec --package api $(openapi) > internal/api/spec.gen.go
 
 build:
-	CGO_ENABLED=0 go build -o bin/api .
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/api .
 
 build-debug:
 	CGO_ENABLED=1 go build -race -gcflags=all="-N -l" -o bin/api .

packages/api/go-templates/fc-build-envs.hcl

@@ -29,12 +29,12 @@ job "{{ .JobName }}/{{ .CodeSnippetID }}" {
       driver = "raw_exec"
 
       artifact {
-        source = "https://storage.googleapis.com/devbook-environment-pipeline/env.tar.gz"
+        source = "https://storage.googleapis.com/e2b-fc-env-pipeline/env.tar.gz"
         destination = "local"
       }
 
       artifact {
-        source = "https://storage.googleapis.com/devbook-environment-pipeline/devbookd"
+        source = "https://storage.googleapis.com/e2b-fc-env-pipeline/devbookd"
         destination = "local/devbookd"
         mode = "file"
       }
@@ -89,7 +89,7 @@ if [ -z "$API_KEY" ]; then
   exit 1
 fi
 
-API_URL="https://ondevbook.com"
+API_URL="https://api.e2b.dev"
 ENVS_ENDPOINT="${API_URL}/envs/${CODE_SNIPPET_ID}/state?api_key=$API_KEY"
 
 # Main didn't finish successfully.

packages/api/go-templates/fc-update-envs.hcl

@@ -30,7 +30,7 @@ job "{{ .JobName }}/{{ .CodeSnippetID }}" {
       }
 
       artifact {
-        source = "https://storage.googleapis.com/devbook-environment-pipeline/env.tar.gz"
+        source = "https://storage.googleapis.com/e2b-fc-env-pipeline/env.tar.gz"
         destination = "local"
       }
 
@@ -83,7 +83,7 @@ if [ -z "$API_KEY" ]; then
   exit 1
 fi
 
-API_URL="https://ondevbook.com"
+API_URL="https://api.e2b.dev"
 ENVS_ENDPOINT="${API_URL}/envs/${CODE_SNIPPET_ID}/state?api_key=$API_KEY"
 
 # Main didn't finish successfully.

packages/api/go-templates/fc-use-prebuilt-envs.hcl

@@ -30,7 +30,7 @@ job "{{ .JobName }}/{{ .CodeSnippetID }}" {
       }
 
       artifact {
-        source = "https://storage.googleapis.com/devbook-environment-pipeline/env.tar.gz"
+        source = "https://storage.googleapis.com/e2b-fc-env-pipeline/env.tar.gz"
         destination = "local"
       }
 
@@ -81,7 +81,7 @@ if [ -z "$API_KEY" ]; then
   exit 1
 fi
 
-API_URL="https://ondevbook.com"
+API_URL="https://api.e2b.dev"
 ENVS_ENDPOINT="${API_URL}/envs/${CODE_SNIPPET_ID}/state?api_key=$API_KEY"
 
 # Main didn't finish successfully.

packages/api/internal/api/key.go

@@ -2,8 +2,4 @@ package api
 
 import "os"
 
-const (
-	APIAdminKeyName = "API_ADMIN_KEY"
-)
-
-var APIAdminKey = os.Getenv(APIAdminKeyName)
+var APIAdminKey = os.Getenv("API_ADMIN_KEY")

packages/api/internal/handlers/sessions.go

@@ -3,6 +3,7 @@ package handlers
 import (
 	"fmt"
 	"net/http"
+	"time"
 
 	"github.com/devbookhq/devbook-api/packages/api/internal/api"
 	"github.com/gin-gonic/gin"
@@ -102,14 +103,14 @@ func (a *APIStore) PostSessions(
 	}
 
 	var teamID *string
-	if a.isPredefinedTemplate(newSession.CodeSnippetID) {
+	if a.isPredefinedTemplate(newSession.CodeSnippetID)  {
 		teamID = a.validateTeamAPIKey(params.ApiKey)
-		if teamID == nil && params.ApiKey != nil {
+		if teamID == nil {
 			_, _, userErr := a.validateAPIKey(params.ApiKey)
 			if userErr != nil {
 				errMsg := fmt.Errorf("invalid API key: %+v", params.ApiKey)
 				ReportCriticalError(ctx, errMsg)
-				a.sendAPIStoreError(c, 401, "Invalid API key")
+				a.sendAPIStoreError(c, 401, "Invalid API key, please visit https://e2b.dev/docs?reason=sdk-missing-api-key to get your API key.")
 				return
 			}
 		}
@@ -122,6 +123,7 @@ func (a *APIStore) PostSessions(
 		a.sendAPIStoreError(c, err.Code, err.ClientMsg)
 		return
 	}
+	startTime := time.Now()
 	ReportEvent(ctx, "created session")
 	if teamID != nil {
 		err := a.posthog.Enqueue(posthog.GroupIdentify{
@@ -148,7 +150,6 @@ func (a *APIStore) PostSessions(
 		if err != nil {
 			fmt.Printf("Error when sending event to Posthog: %+v\n", err)
 		}
-		a.teamCache.Add(session.SessionID, *teamID)
 	}
 
 	if *newSession.EditEnabled {
@@ -175,7 +176,7 @@ func (a *APIStore) PostSessions(
 		}
 	}
 
-	if err := a.sessionsCache.Add(session); err != nil {
+	if err := a.sessionsCache.Add(session, teamID, &startTime); err != nil {
 		errMsg := fmt.Errorf("error when adding session to cache: %v", err)
 		ReportError(ctx, errMsg)