diff --git a/.github/disabled-workflows/devbookd.yml b/.github/disabled-workflows/devbookd.yml index 9842e88981..00718a46a9 100644 --- a/.github/disabled-workflows/devbookd.yml +++ b/.github/disabled-workflows/devbookd.yml @@ -42,7 +42,7 @@ jobs: service_account: ${{ secrets.service_account_email }} - name: Upload devbookd - uses: 'google-github-actions/upload-cloud-storage@v1' + uses: "google-github-actions/upload-cloud-storage@v1" with: - path: './packages/devbookd/dist/devbookd_linux_amd64_v1/devbookd' - destination: 'devbook-environment-pipeline' + path: "./packages/devbookd/dist/devbookd_linux_amd64_v1/devbookd" + destination: "e2b-fc-env-pipeline" diff --git a/.github/disabled-workflows/fc-env.yml b/.github/disabled-workflows/fc-env.yml index 8785db5e6d..f0167488db 100644 --- a/.github/disabled-workflows/fc-env.yml +++ b/.github/disabled-workflows/fc-env.yml @@ -27,7 +27,7 @@ jobs: service_account: ${{ secrets.service_account_email }} - name: Upload env pipelines - uses: 'google-github-actions/upload-cloud-storage@v1' + uses: "google-github-actions/upload-cloud-storage@v1" with: - path: './packages/fc-env/env.tar.gz' - destination: 'devbook-environment-pipeline' + path: "./packages/fc-env/env.tar.gz" + destination: "e2b-fc-env-pipeline" diff --git a/.github/disabled-workflows/firecracker-task-driver.yml b/.github/disabled-workflows/firecracker-task-driver.yml index 6918e3bb8c..883608bb45 100644 --- a/.github/disabled-workflows/firecracker-task-driver.yml +++ b/.github/disabled-workflows/firecracker-task-driver.yml @@ -16,14 +16,13 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - - name: Set up Go uses: actions/setup-go@v3 with: go-version-file: ./packages/firecracker-task-driver/go.mod cache: true cache-dependency-path: ./packages/firecracker-task-driver/go.sum - + - name: Download deps working-directory: ./packages/firecracker-task-driver run: go mod tidy @@ -39,7 +38,7 @@ jobs: service_account: ${{ secrets.service_account_email }} - name: Upload firecracker task driver - uses: 'google-github-actions/upload-cloud-storage@v1' + uses: "google-github-actions/upload-cloud-storage@v1" with: - path: './packages/firecracker-task-driver/bin/firecracker-task-driver' - destination: 'devbook-environment-pipeline' \ No newline at end of file + path: "./packages/firecracker-task-driver/bin/firecracker-task-driver" + destination: "e2b-fc-env-pipeline" diff --git a/.gitignore b/.gitignore index 3b0fde4cc2..72e1c5f58f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ core.3071 node_modules # Ignore Vale.sh styles directory vale-styles + +.env diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index d963cfec8e..948f272812 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -4,7 +4,6 @@ provider "registry.terraform.io/hashicorp/consul" { version = "2.18.0" hashes = [ - "h1:ABHNF3vYoRej5vLiiKR/XYgjhjubsjlVROssc555Vj8=", "h1:GIbc9IimARunBuk+GdxgUX0cnZ2WwFW0IZMyT8E88B8=", "zh:155f6899c8f1e0162169e4d09f426ea9a9738d74129f0ffc54b7570941c49cd4", "zh:241ba909b387c349845b0371321af45ccbb00290edbe1ea0861ad8732951f718", @@ -67,7 +66,6 @@ provider "registry.terraform.io/hashicorp/nomad" { version = "1.4.20" hashes = [ "h1:4IKRa9RKAwJ55lLQjgkOkXLnm+KV4T0bCH+ldNhRR14=", - "h1:M/QVXHPfeySejJZI3I8mBYrL/J9VsbnyF/dKIMlUhXo=", "zh:02989edcebe724fc0aa873b22176fd20074c4f46295e728010711a8fc5dfa72c", "zh:089ba7d19bcf5c6bab3f8b8c5920eb6d78c52cf79bb0c5dfeb411c600e7efcba", "zh:235865a2182ca372bcbf440201a8b8cc0715ad5dbc4de893d99b6f32b5be53ab", diff --git a/DEV.md b/DEV.md index 47702f47e2..bf4874e71c 100644 --- a/DEV.md +++ b/DEV.md @@ -2,8 +2,6 @@ ## Infra -- Clock drift - - https://github.com/firecracker-microvm/firecracker/blob/eb8de3ba1f7cb636d2aaa632fe96b234f3a302e6/FAQ.md#my-guest-wall-clock-is-drifting-how-can-i-fix-it - Change read/write file to allow other thatn utf-8 format so we don't break the files - Remove dnsmasq from cluster VM image if it is not needed - Remove need for provisioning during building env @@ -23,8 +21,8 @@ - Check if the kernel args we are using are what we want - Improve API (efficient calls Nomad calls, no polling, etc.) - Improve FC WS connection (subscriptions take additional calls, maybe we can improve that, etc.) -- Move to ubuntu - Update kernel version +- devbookd jsonrpc parameters could be objects instead of arrays (compatibility advantages) - Move to a better FS handling (filestore, fuse?) - Fix possible mutex problems in the firecracker task driver - Update devbookd in all envs automatically @@ -53,17 +51,14 @@ - Use the FC jailer properly - Rebuild only the changed templates on push - Update NodeJS version -- Remove top package-lock - Remove gitpod deps and mentions (makefile, gitpod yaml) - Remove devbook specific code from the repo (allow move to new org or on prem deploy) - Improve caching of the GH actions - Add instance sizes and other things as variables in the GH actions and remove them from codes - API URL should be configurable - add testing/staging env -- create proper monorepo for the packages (pnpm for Node) and improve release system - Add memory swap to Rust - OoM when compiling bigger programs - https://www.digitalocean.com/community/tutorials/how-to-add-swap-space-on-ubuntu-22-04 -- Should we use ubuntu 22 for envs? - Does ubuntu need the newer kernel? - Using driver with multiple kernels - Cloud Hypervisor insted of FC - https://github.com/cloud-hypervisor/cloud-hypervisor @@ -127,6 +122,7 @@ - Add sync support to the python SDK and js sdk - the ws pong is working now but the actuall message backpressure can still make problems - Add golangci-lint +- use dnsmasq instead of /etc/hosts and second proxy ### Python SDK @@ -145,7 +141,7 @@ - add tests - support lower python versions - Check if stderr and stdout are ordered -- Myabe remove types (and timestamps?) from stdout/stderr +- Myabe remove types (and timestamps?) from stdout/stderr (they are already fully identified by the subscription) - Use specific version (nodejs20) for the templates - Using pathlib for paths? windows support? - Flush all stdout/err after killing process or terminal in devbookd and also wait for the Stdout/err in the SDK diff --git a/ENVS.md b/ENVS.md new file mode 100644 index 0000000000..124af07a39 --- /dev/null +++ b/ENVS.md @@ -0,0 +1,8 @@ +# Custom Envs + +- How to create a custom env from dockerfile if you want to use local files during the build step? + - It's either you build the dockerfile locally or you package and upload the files to our infra so it can be used during the build step. + + + +- Can we has the dockerfile+files+scripts/config to get the unique env hash that we can use to start identical envs in the future? diff --git a/Makefile b/Makefile index 10b3c230d0..0b2ec6c0a1 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ login-gcloud: .PHONY: login-gcloud-user login-gcloud-user: gcloud auth login - gcloud config set project devbookhq + gcloud config set project e2b-prod gcloud --quiet auth configure-docker us-central1-docker.pkg.dev .PHONY: init diff --git a/main.tf b/main.tf index d4fc4e1c6b..6e046bf8e5 100644 --- a/main.tf +++ b/main.tf @@ -1,7 +1,7 @@ terraform { required_version = ">=1.1.9" backend "gcs" { - bucket = "devbook-terraform-state" + bucket = "e2b-terraform-state" prefix = "terraform/orchestration/state" } required_providers { @@ -77,7 +77,7 @@ resource "consul_acl_token_policy_attachment" "attachment" { } data "google_compute_global_address" "orch_server_ip" { - name = "orch-server-ip" + name = "orch-server-nomad-ip" } data "google_secret_manager_secret_version" "nomad_acl_token" { @@ -93,18 +93,18 @@ data "google_secret_manager_secret_version" "lightstep_api_key" { secret = "lightstep-api-key" } -data "google_secret_manager_secret_version" "logtail_api_key" { - secret = "logtail-api-key" +data "google_secret_manager_secret_version" "betterstack_logs_api_key" { + secret = "betterstack-logs-api-key" } module "telemetry" { source = "./packages/telemetry" - logs_health_proxy_port = var.logs_health_proxy_port - logs_proxy_port = var.logs_proxy_port - lightstep_api_key = data.google_secret_manager_secret_version.lightstep_api_key.secret_data - logtail_api_key = data.google_secret_manager_secret_version.logtail_api_key.secret_data - gcp_zone = var.gcp_zone + logs_health_proxy_port = var.logs_health_proxy_port + logs_proxy_port = var.logs_proxy_port + lightstep_api_key = data.google_secret_manager_secret_version.lightstep_api_key.secret_data + betterstack_logs_api_key = data.google_secret_manager_secret_version.betterstack_logs_api_key.secret_data + gcp_zone = var.gcp_zone } module "session_proxy" { diff --git a/openapi.yml b/openapi.yml index 7e73c779bb..b3eae2914f 100644 --- a/openapi.yml +++ b/openapi.yml @@ -1,13 +1,12 @@ openapi: 3.0.0 info: - version: 1.0.0 - title: Devbook - description: Devbook API + version: 0.1.0 + title: E2B API servers: - - url: https://ondevbook.com/ + - url: https://api.e2b.dev/ description: API endpoint - - url: https://{sessionID}-{clientID}.ondevbook.com/ + - url: https://{sessionID}-{clientID}.api.e2b.dev/ description: Session endpoint without a specified port variables: sessionID: @@ -16,7 +15,7 @@ servers: clientID: description: ID of the client default: _clientID - - url: https://{port}-{sessionID}-{clientID}.ondevbook.com/ + - url: https://{port}-{sessionID}-{clientID}.api.e2b.dev/ description: Session endpoint with a specificed port variables: sessionID: diff --git a/packages/api/Dockerfile b/packages/api/Dockerfile index 473246be64..893d574778 100644 --- a/packages/api/Dockerfile +++ b/packages/api/Dockerfile @@ -22,4 +22,3 @@ RUN chmod +x api # Set Gin server to the production mode ENV GIN_MODE=release ENTRYPOINT [ "./api"] - \ No newline at end of file diff --git a/packages/api/Makefile b/packages/api/Makefile index 1a6e5daf75..3520cc2d77 100644 --- a/packages/api/Makefile +++ b/packages/api/Makefile @@ -1,6 +1,6 @@ openapi := ../../openapi.yml -GCP_PROJECT := devbookhq +GCP_PROJECT := e2b-prod IMAGE := orchestration/api generate: @@ -9,7 +9,7 @@ generate: oapi-codegen -old-config-style -generate spec --package api $(openapi) > internal/api/spec.gen.go build: - CGO_ENABLED=0 go build -o bin/api . + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/api . build-debug: CGO_ENABLED=1 go build -race -gcflags=all="-N -l" -o bin/api . diff --git a/packages/api/go-templates/fc-build-envs.hcl b/packages/api/go-templates/fc-build-envs.hcl index 19831b9484..8eee2b763d 100644 --- a/packages/api/go-templates/fc-build-envs.hcl +++ b/packages/api/go-templates/fc-build-envs.hcl @@ -29,12 +29,12 @@ job "{{ .JobName }}/{{ .CodeSnippetID }}" { driver = "raw_exec" artifact { - source = "https://storage.googleapis.com/devbook-environment-pipeline/env.tar.gz" + source = "https://storage.googleapis.com/e2b-fc-env-pipeline/env.tar.gz" destination = "local" } artifact { - source = "https://storage.googleapis.com/devbook-environment-pipeline/devbookd" + source = "https://storage.googleapis.com/e2b-fc-env-pipeline/devbookd" destination = "local/devbookd" mode = "file" } @@ -89,7 +89,7 @@ if [ -z "$API_KEY" ]; then exit 1 fi -API_URL="https://ondevbook.com" +API_URL="https://api.e2b.dev" ENVS_ENDPOINT="${API_URL}/envs/${CODE_SNIPPET_ID}/state?api_key=$API_KEY" # Main didn't finish successfully. diff --git a/packages/api/go-templates/fc-update-envs.hcl b/packages/api/go-templates/fc-update-envs.hcl index 065265cc60..6ec9c00a45 100644 --- a/packages/api/go-templates/fc-update-envs.hcl +++ b/packages/api/go-templates/fc-update-envs.hcl @@ -30,7 +30,7 @@ job "{{ .JobName }}/{{ .CodeSnippetID }}" { } artifact { - source = "https://storage.googleapis.com/devbook-environment-pipeline/env.tar.gz" + source = "https://storage.googleapis.com/e2b-fc-env-pipeline/env.tar.gz" destination = "local" } @@ -83,7 +83,7 @@ if [ -z "$API_KEY" ]; then exit 1 fi -API_URL="https://ondevbook.com" +API_URL="https://api.e2b.dev" ENVS_ENDPOINT="${API_URL}/envs/${CODE_SNIPPET_ID}/state?api_key=$API_KEY" # Main didn't finish successfully. diff --git a/packages/api/go-templates/fc-use-prebuilt-envs.hcl b/packages/api/go-templates/fc-use-prebuilt-envs.hcl index f58ddc8b0a..00323f7f2d 100644 --- a/packages/api/go-templates/fc-use-prebuilt-envs.hcl +++ b/packages/api/go-templates/fc-use-prebuilt-envs.hcl @@ -30,7 +30,7 @@ job "{{ .JobName }}/{{ .CodeSnippetID }}" { } artifact { - source = "https://storage.googleapis.com/devbook-environment-pipeline/env.tar.gz" + source = "https://storage.googleapis.com/e2b-fc-env-pipeline/env.tar.gz" destination = "local" } @@ -81,7 +81,7 @@ if [ -z "$API_KEY" ]; then exit 1 fi -API_URL="https://ondevbook.com" +API_URL="https://api.e2b.dev" ENVS_ENDPOINT="${API_URL}/envs/${CODE_SNIPPET_ID}/state?api_key=$API_KEY" # Main didn't finish successfully. diff --git a/packages/api/internal/api/key.go b/packages/api/internal/api/key.go index 9b939e3529..5c7fb64d81 100644 --- a/packages/api/internal/api/key.go +++ b/packages/api/internal/api/key.go @@ -2,8 +2,4 @@ package api import "os" -const ( - APIAdminKeyName = "API_ADMIN_KEY" -) - -var APIAdminKey = os.Getenv(APIAdminKeyName) +var APIAdminKey = os.Getenv("API_ADMIN_KEY") diff --git a/packages/api/internal/handlers/sessions.go b/packages/api/internal/handlers/sessions.go index b27f7d1ea5..830ba5fe2b 100644 --- a/packages/api/internal/handlers/sessions.go +++ b/packages/api/internal/handlers/sessions.go @@ -3,6 +3,7 @@ package handlers import ( "fmt" "net/http" + "time" "github.com/devbookhq/devbook-api/packages/api/internal/api" "github.com/gin-gonic/gin" @@ -102,14 +103,14 @@ func (a *APIStore) PostSessions( } var teamID *string - if a.isPredefinedTemplate(newSession.CodeSnippetID) { + if a.isPredefinedTemplate(newSession.CodeSnippetID) { teamID = a.validateTeamAPIKey(params.ApiKey) - if teamID == nil && params.ApiKey != nil { + if teamID == nil { _, _, userErr := a.validateAPIKey(params.ApiKey) if userErr != nil { errMsg := fmt.Errorf("invalid API key: %+v", params.ApiKey) ReportCriticalError(ctx, errMsg) - a.sendAPIStoreError(c, 401, "Invalid API key") + a.sendAPIStoreError(c, 401, "Invalid API key, please visit https://e2b.dev/docs?reason=sdk-missing-api-key to get your API key.") return } } @@ -122,6 +123,7 @@ func (a *APIStore) PostSessions( a.sendAPIStoreError(c, err.Code, err.ClientMsg) return } + startTime := time.Now() ReportEvent(ctx, "created session") if teamID != nil { err := a.posthog.Enqueue(posthog.GroupIdentify{ @@ -148,7 +150,6 @@ func (a *APIStore) PostSessions( if err != nil { fmt.Printf("Error when sending event to Posthog: %+v\n", err) } - a.teamCache.Add(session.SessionID, *teamID) } if *newSession.EditEnabled { @@ -175,7 +176,7 @@ func (a *APIStore) PostSessions( } } - if err := a.sessionsCache.Add(session); err != nil { + if err := a.sessionsCache.Add(session, teamID, &startTime); err != nil { errMsg := fmt.Errorf("error when adding session to cache: %v", err) ReportError(ctx, errMsg) diff --git a/packages/api/internal/handlers/store.go b/packages/api/internal/handlers/store.go index 3ed9b2043d..9e39da90f2 100644 --- a/packages/api/internal/handlers/store.go +++ b/packages/api/internal/handlers/store.go @@ -19,7 +19,6 @@ import ( type APIStore struct { sessionsCache *nomad.SessionCache - teamCache *nomad.TeamCache nomad *nomad.NomadClient supabase *supabase.DB NextId int64 @@ -29,6 +28,8 @@ type APIStore struct { posthog posthog.Client } +var posthogAPIKey = os.Getenv("POSTHOG_API_KEY") + func NewAPIStore() *APIStore { fmt.Println("Initializing API store") @@ -63,13 +64,8 @@ func NewAPIStore() *APIStore { fmt.Fprintf(os.Stderr, "Error loading current sessions from Nomad\n: %s", sessionErr) } - cache := nomad.NewSessionCache(nomadClient.DeleteSession, initialSessions) - teamCache := nomad.NewTeamCache() - // Comment this line out if you are developing locally to prevent killing sessions in production - go cache.KeepInSync(nomadClient) - posthogAPIKey := os.Getenv("POSTHOG_API_KEY") - client, posthogErr := posthog.NewWithConfig(posthogAPIKey, posthog.Config{ + posthogClient, posthogErr := posthog.NewWithConfig(posthogAPIKey, posthog.Config{ Interval: 30 * time.Second, BatchSize: 100, Verbose: true, @@ -80,15 +76,18 @@ func NewAPIStore() *APIStore { panic(posthogErr) } + cache := nomad.NewSessionCache(getDeleteSessionFunction(nomadClient, posthogClient), initialSessions) + // Comment this line out if you are developing locally to prevent killing sessions in production + go cache.KeepInSync(nomadClient) + return &APIStore{ nomad: nomadClient, supabase: supabaseClient, NextId: 1000, sessionsCache: cache, - teamCache: teamCache, tracer: tracer, templates: templates, - posthog: client, + posthog: posthogClient, } } @@ -178,7 +177,19 @@ func (a *APIStore) validateTeamAPIKey(apiKey *string) *string { } func (a *APIStore) DeleteSession(sessionID string, purge bool) *api.APIError { - duration, err := a.nomad.DeleteSessionWithDuration(sessionID, purge) + sessionData := a.sessionsCache.Get(sessionID) + return deleteSession(a.nomad, a.posthog, sessionID, sessionData.TeamID, sessionData.StartTime, purge) +} + +type sessionData = nomad.SessionData + +func getDeleteSessionFunction(nomad *nomad.NomadClient, posthogClient posthog.Client) func(sessionData sessionData, purge bool) *api.APIError { + return func(sessionData sessionData, purge bool) *api.APIError { + return deleteSession(nomad, posthogClient, sessionData.Session.SessionID, sessionData.TeamID, sessionData.StartTime, purge) + } +} +func deleteSession(nomad *nomad.NomadClient, posthogClient posthog.Client, sessionID string, teamID *string, startTime *time.Time, purge bool) *api.APIError { + err := nomad.DeleteSession(sessionID, purge) if err != nil { return &api.APIError{ Msg: fmt.Sprintf("cannot delete session '%s': %+v", sessionID, err), @@ -186,13 +197,13 @@ func (a *APIStore) DeleteSession(sessionID string, purge bool) *api.APIError { Code: http.StatusInternalServerError, } } - teamID, teamErr := a.teamCache.Get(sessionID) - if teamErr != nil { - err := a.posthog.Enqueue(posthog.Capture{ - DistinctId: "backend_infra", + + if teamID != nil && startTime != nil { + err := posthogClient.Enqueue(posthog.Capture{ + DistinctId: "backend", Event: "closed_session", Properties: posthog.NewProperties(). - Set("session_id", sessionID).Set("duration", duration), + Set("session_id", sessionID).Set("duration", time.Since(*startTime).Seconds()), Groups: posthog.NewGroups(). Set("team", teamID), }) diff --git a/packages/api/internal/nomad/cache.go b/packages/api/internal/nomad/cache.go index 3c7f45a14b..e245c312d8 100644 --- a/packages/api/internal/nomad/cache.go +++ b/packages/api/internal/nomad/cache.go @@ -15,17 +15,27 @@ const ( cacheSyncTime = time.Second * 180 ) +type SessionData struct { + Session *api.Session + TeamID *string + StartTime *time.Time +} + type SessionCache struct { - cache *ttlcache.Cache[string, *api.Session] + cache *ttlcache.Cache[string, SessionData] } // Add the session to the cache and start expiration timer -func (c *SessionCache) Add(session *api.Session) error { +func (c *SessionCache) Add(session *api.Session, teamID *string, startTime *time.Time) error { if c.Exists(session.SessionID) { return fmt.Errorf("session \"%s\" already exists", session.SessionID) } - - c.cache.Set(session.SessionID, session, ttlcache.DefaultTTL) + sessionData := SessionData{ + Session: session, + TeamID: teamID, + StartTime: startTime, + } + c.cache.Set(session.SessionID, sessionData, ttlcache.DefaultTTL) return nil } @@ -39,21 +49,31 @@ func (c *SessionCache) Refresh(sessionID string) error { return nil } +// Get the session from the cache +func (c *SessionCache) Get(sessionID string) SessionData { + item := c.cache.Get(sessionID, ttlcache.WithDisableTouchOnHit[string, SessionData]()) + if item != nil { + return item.Value() + } else { + panic(fmt.Errorf("session \"%s\" doesn't exist", sessionID)) + } +} + // Check if the session exists in the cache func (c *SessionCache) Exists(sessionID string) bool { - item := c.cache.Get(sessionID, ttlcache.WithDisableTouchOnHit[string, *api.Session]()) + item := c.cache.Get(sessionID, ttlcache.WithDisableTouchOnHit[string, SessionData]()) return item != nil } func (c *SessionCache) FindEditSession(codeSnippetID string) (*api.Session, error) { for _, item := range c.cache.Items() { - if item.Value() == nil { + if item.Value().Session == nil { continue } - if item.Value().EditEnabled && item.Value().CodeSnippetID == codeSnippetID { + if item.Value().Session.EditEnabled && item.Value().Session.CodeSnippetID == codeSnippetID { c.Refresh(item.Key()) - return item.Value(), nil + return item.Value().Session, nil } } return nil, fmt.Errorf("error edit session for code snippet '%s' not found", codeSnippetID) @@ -62,21 +82,21 @@ func (c *SessionCache) FindEditSession(codeSnippetID string) (*api.Session, erro func (c *SessionCache) Sync(sessions []*api.Session) { for _, session := range sessions { if !c.Exists(session.SessionID) { - c.Add(session) + c.Add(session, nil, nil) } } } // We will need to either use Redis for storing active sessions OR retrieve them from Nomad when we start API to keep everything in sync // We are retrieving the tasks from Nomad now -func NewSessionCache(handleDeleteSession func(sessionID string, purge bool) *api.APIError, initialSessions []*api.Session) *SessionCache { +func NewSessionCache(handleDeleteSession func(sessionData SessionData, purge bool) *api.APIError, initialSessions []*api.Session) *SessionCache { cache := ttlcache.New( - ttlcache.WithTTL[string, *api.Session](sessionExpiration), + ttlcache.WithTTL[string, SessionData](sessionExpiration), ) - cache.OnEviction(func(ctx context.Context, er ttlcache.EvictionReason, i *ttlcache.Item[string, *api.Session]) { + cache.OnEviction(func(ctx context.Context, er ttlcache.EvictionReason, i *ttlcache.Item[string, SessionData]) { if er == ttlcache.EvictionReasonExpired || er == ttlcache.EvictionReasonDeleted { - err := handleDeleteSession(i.Key(), true) + err := handleDeleteSession(i.Value(), true) if err != nil { fmt.Fprintf(os.Stderr, "Error deleting session (%v)\n: %s", er, err.Error()) } @@ -88,7 +108,7 @@ func NewSessionCache(handleDeleteSession func(sessionID string, purge bool) *api } for _, session := range initialSessions { - sessionCache.Add(session) + sessionCache.Add(session, nil, nil) } go cache.Start() @@ -112,38 +132,3 @@ func (c *SessionCache) KeepInSync(client *NomadClient) { func (c *SessionCache) Count() int { return c.cache.Len() } - -type TeamCache struct { - cache *ttlcache.Cache[string, string] -} - -func (c *TeamCache) Add(sessionID string, teamID string) { - c.cache.Set(sessionID, teamID, ttlcache.DefaultTTL) -} - -// Check if the session exists in the cache -func (c *TeamCache) Exists(sessionID string) bool { - item := c.cache.Get(sessionID, ttlcache.WithDisableTouchOnHit[string, string]()) - return item != nil -} - -func (c *TeamCache) Get(sessionID string) (string, error) { - item := c.cache.Get(sessionID) - if item == nil { - return "", fmt.Errorf("no team for the session \"%s\" found", sessionID) - - } - return item.Value(), nil -} -func NewTeamCache() *TeamCache { - cache := ttlcache.New( - ttlcache.WithTTL[string, string](sessionExpiration), - ) - - teamCache := &TeamCache{ - cache: cache, - } - go cache.Start() - - return teamCache -} diff --git a/packages/api/internal/nomad/client.go b/packages/api/internal/nomad/client.go index 3f055d7c80..d05413d8f8 100644 --- a/packages/api/internal/nomad/client.go +++ b/packages/api/internal/nomad/client.go @@ -10,14 +10,19 @@ import ( "github.com/hashicorp/nomad/api" ) +var ( + nomadAddress = os.Getenv("NOMAD_ADDRESS") + nomadToken = os.Getenv("NOMAD_TOKEN") +) + type NomadClient struct { client *api.Client } func InitNomadClient() *NomadClient { config := api.Config{ - Address: os.Getenv("NOMAD_ADDRESS"), - SecretID: os.Getenv("NOMAD_TOKEN"), + Address: nomadAddress, + SecretID: nomadToken, } client, err := api.NewClient(&config) diff --git a/packages/api/internal/nomad/session.go b/packages/api/internal/nomad/session.go index 40dff09451..5e016beb62 100644 --- a/packages/api/internal/nomad/session.go +++ b/packages/api/internal/nomad/session.go @@ -188,15 +188,3 @@ func (n *NomadClient) DeleteSession(sessionID string, purge bool) *api.APIError } return nil } - -func (n *NomadClient) DeleteSessionWithDuration(sessionID string, purge bool) (*time.Duration, *api.APIError) { - _, meta, err := n.client.Jobs().Deregister(sessionsJobNameWithSlash+sessionID, purge, &nomadAPI.WriteOptions{}) - if err != nil { - return nil, &api.APIError{ - Msg: fmt.Sprintf("cannot delete job '%s%s' job: %+v", sessionsJobNameWithSlash, sessionID, err), - ClientMsg: "Cannot delete the session right now", - Code: http.StatusInternalServerError, - } - } - return &meta.RequestTime, nil -} diff --git a/packages/api/internal/supabase/client.go b/packages/api/internal/supabase/client.go index 0a1f79fcbf..38a9c36309 100644 --- a/packages/api/internal/supabase/client.go +++ b/packages/api/internal/supabase/client.go @@ -13,9 +13,12 @@ type DB struct { Client *postgrest.Client } +var ( + supabaseURL = os.Getenv("SUPABASE_URL") + supabaseKey = os.Getenv("SUPABASE_KEY") +) + func NewClient() (*DB, error) { - supabaseURL := os.Getenv("SUPABASE_URL") - supabaseKey := os.Getenv("SUPABASE_KEY") parsedURL, err := url.Parse(supabaseURL) if err != nil { diff --git a/packages/api/variables.tf b/packages/api/variables.tf index 6f013db78c..ba1c819ada 100644 --- a/packages/api/variables.tf +++ b/packages/api/variables.tf @@ -25,7 +25,7 @@ variable "api_port" { variable "image_name" { type = string - default = "us-central1-docker.pkg.dev/devbookhq/orchestration/api" + default = "us-central1-docker.pkg.dev/e2b-prod/orchestration/api" } variable "logs_proxy_address" { diff --git a/packages/client-proxy/client-proxy.hcl b/packages/client-proxy/client-proxy.hcl index 8f06e9e615..4274fffc31 100644 --- a/packages/client-proxy/client-proxy.hcl +++ b/packages/client-proxy/client-proxy.hcl @@ -92,7 +92,7 @@ server { [[ range service "session-proxy" ]] server { listen 3002; - server_name ~^(.+)-[[ index .ServiceMeta "Client" | sprig_substr 0 8 ]]\.ondevbook\.com$; + server_name ~^(.+)-[[ index .ServiceMeta "Client" | sprig_substr 0 8 ]]\.api\.e2b\.dev$; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; diff --git a/packages/cluster-disk-image/main.pkr.hcl b/packages/cluster-disk-image/main.pkr.hcl index ffd500ca38..d1a79a046e 100644 --- a/packages/cluster-disk-image/main.pkr.hcl +++ b/packages/cluster-disk-image/main.pkr.hcl @@ -1,5 +1,5 @@ packer { - required_version = "1.8.4" + required_version = ">=1.8.4" required_plugins { googlecompute = { version = "1.0.16" @@ -52,7 +52,7 @@ build { provisioner "shell" { inline = [ "sudo apt-get update", - "sudo apt-get install -y docker.io=20.10.12-0ubuntu2~20.04.1", + "sudo apt-get install -y docker.io", "sudo systemctl start docker", "sudo usermod -aG docker $USER", ] @@ -90,7 +90,7 @@ build { provisioner "shell" { inline = [ "sudo mkdir -p /opt/nomad/plugins", - "sudo curl https://storage.googleapis.com/devbook-environment-pipeline/firecracker-task-driver -o /opt/nomad/plugins/firecracker-task-driver", + "sudo curl https://storage.googleapis.com/e2b-fc-env-pipeline/firecracker-task-driver -o /opt/nomad/plugins/firecracker-task-driver", "sudo chmod +x /opt/nomad/plugins/firecracker-task-driver", ] } @@ -98,7 +98,7 @@ build { provisioner "shell" { inline = [ "sudo mkdir -p /fc-vm", - "sudo curl https://storage.googleapis.com/devbook-snapshot/vmlinux.bin -o /fc-vm/vmlinux.bin", + "sudo curl https://storage.googleapis.com/e2b-fc-env-pipeline/vmlinux.bin -o /fc-vm/vmlinux.bin", ] } } diff --git a/packages/cluster-disk-image/variables.pkr.hcl b/packages/cluster-disk-image/variables.pkr.hcl index 1fd898a5f1..d3a82f8a54 100644 --- a/packages/cluster-disk-image/variables.pkr.hcl +++ b/packages/cluster-disk-image/variables.pkr.hcl @@ -1,11 +1,11 @@ variable "gcp_project_id" { type = string - default = "devbookhq" + default = "e2b-prod" } variable "gcp_zone" { type = string - default = "us-central1-a" + default = "us-central1-c" } variable "consul_version" { diff --git a/packages/cluster/client/main.tf b/packages/cluster/client/main.tf index 5abb7a702a..06aba4bea6 100644 --- a/packages/cluster/client/main.tf +++ b/packages/cluster/client/main.tf @@ -126,12 +126,12 @@ resource "google_compute_instance_template" "client" { # This cert is for proxying throught Cloudflare only data "google_compute_ssl_certificate" "session_certificate" { - name = "sessions" + name = "e2b-sessions" } # This should be SSL cert for usage withotu Cloudflare -data "google_compute_ssl_certificate" "ondevbook_certificate" { - name = "ondevbook" +data "google_compute_ssl_certificate" "api_certificate" { + name = "e2b-api" } resource "google_compute_url_map" "client_map" { @@ -139,12 +139,12 @@ resource "google_compute_url_map" "client_map" { default_service = module.gce_lb_http.backend_services["session"].self_link host_rule { - hosts = ["ondevbook.com"] + hosts = ["api.e2b.dev"] path_matcher = "api-paths" } host_rule { - hosts = ["*.ondevbook.com"] + hosts = ["*.e2b.dev"] path_matcher = "session-paths" } @@ -159,8 +159,8 @@ resource "google_compute_url_map" "client_map" { } } -data "google_compute_global_address" "orch_client_ip" { - name = "orch-client-ip" +data "google_compute_global_address" "orch_client_api_ip" { + name = "orch-client-api-ip" } module "gce_lb_http" { @@ -168,10 +168,10 @@ module "gce_lb_http" { version = "~> 5.1" name = "orch-external-session" project = var.gcp_project_id - address = data.google_compute_global_address.orch_client_ip.address + address = data.google_compute_global_address.orch_client_api_ip.address ssl_certificates = [ data.google_compute_ssl_certificate.session_certificate.self_link, - data.google_compute_ssl_certificate.ondevbook_certificate.self_link, + data.google_compute_ssl_certificate.api_certificate.self_link, ] create_address = false use_ssl_certificates = true diff --git a/packages/cluster/server/main.tf b/packages/cluster/server/main.tf index b341e16c07..46d9ba702b 100644 --- a/packages/cluster/server/main.tf +++ b/packages/cluster/server/main.tf @@ -116,7 +116,11 @@ resource "google_compute_instance_template" "server" { # LOAD BALANCERS data "google_compute_global_address" "orch_server_ip" { - name = "orch-server-ip" + name = "orch-server-nomad-ip" +} + +data "google_compute_ssl_certificate" "nomad_certificate" { + name = "e2b-nomad-api" } module "gce_lb_http_nomad" { @@ -129,6 +133,13 @@ module "gce_lb_http_nomad" { target_tags = [ var.cluster_tag_name, ] + ssl_certificates = [ + data.google_compute_ssl_certificate.nomad_certificate.self_link, + ] + use_ssl_certificates = true + ssl = true + + firewall_networks = [var.network_name] backends = { @@ -191,6 +202,10 @@ data "google_compute_global_address" "orch_server_consul_ip" { name = "orch-server-consul-ip" } +data "google_compute_ssl_certificate" "consul_certificate" { + name = "e2b-consul-api" +} + module "gce_lb_http_consul" { source = "GoogleCloudPlatform/lb-http/google" version = "~> 5.1" @@ -201,6 +216,13 @@ module "gce_lb_http_consul" { target_tags = [ var.cluster_tag_name, ] + + ssl_certificates = [ + data.google_compute_ssl_certificate.consul_certificate.self_link, + ] + use_ssl_certificates = true + ssl = true + firewall_networks = [var.network_name] backends = { diff --git a/packages/devbookd/upload-local.sh b/packages/devbookd/upload-local.sh index 0f00ed870c..b3b7d4ddca 100755 --- a/packages/devbookd/upload-local.sh +++ b/packages/devbookd/upload-local.sh @@ -3,4 +3,4 @@ set -euo pipefail gsutil -h "Cache-Control:no-cache, max-age=0" \ - cp bin/devbookd gs://devbook-environment-pipeline/devbookd + cp bin/devbookd gs://e2b-fc-env-pipeline/devbookd diff --git a/packages/devbookd/upload.sh b/packages/devbookd/upload.sh index 0195da9aae..1d875c259a 100755 --- a/packages/devbookd/upload.sh +++ b/packages/devbookd/upload.sh @@ -3,4 +3,4 @@ set -euo pipefail gsutil -h "Cache-Control:no-cache, max-age=0" \ - cp dist/devbookd_linux_amd64_v1/devbookd gs://devbook-environment-pipeline/devbookd + cp dist/devbookd_linux_amd64_v1/devbookd gs://e2b-fc-env-pipeline/devbookd diff --git a/packages/fc-env/build-env.sh b/packages/fc-env/build-env.sh index b139c913f8..42cfedf07f 100755 --- a/packages/fc-env/build-env.sh +++ b/packages/fc-env/build-env.sh @@ -57,7 +57,7 @@ if [ -z "$DEVBOOKD" ]; then exit 1 fi -API_URL="https://ondevbook.com" +API_URL="https://api.e2b.dev" ENVS_ENDPOINT="${API_URL}/envs/${CODE_SNIPPET_ID}/state?api_key=$API_KEY" echo "==== Args ===========================================================================================" diff --git a/packages/fc-env/publish.sh b/packages/fc-env/publish.sh index 8683bb05e4..ba38c1a9cb 100755 --- a/packages/fc-env/publish.sh +++ b/packages/fc-env/publish.sh @@ -11,7 +11,7 @@ set -euo pipefail ./package.sh gsutil -h "Cache-Control:no-cache, max-age=0" \ - cp env.tar.gz gs://devbook-environment-pipeline + cp env.tar.gz gs://e2b-fc-env-pipeline rm env.tar.gz rm -rf env diff --git a/packages/fc-env/update-env.sh b/packages/fc-env/update-env.sh index 9157b10883..8a2182e2d5 100755 --- a/packages/fc-env/update-env.sh +++ b/packages/fc-env/update-env.sh @@ -46,7 +46,7 @@ if [ -z "$API_KEY" ]; then exit 1 fi -API_URL="https://ondevbook.com" +API_URL="https://api.e2b.dev" ENVS_ENDPOINT="${API_URL}/envs/${CODE_SNIPPET_ID}/state?api_key=$API_KEY" echo "==== Args ===========================================================================================" @@ -89,7 +89,7 @@ function mkdirs() { } function get_snapshot() { - EDIT_ID=`cat $EDIT_ID_PATH` + EDIT_ID=$(cat $EDIT_ID_PATH) EDIT_ID_DIR="$EDIT_DIR/$EDIT_ID" if [[ "$SESSION_ID" == "" ]]; then @@ -104,18 +104,18 @@ function get_snapshot() { echo "Running edit sessions rootfs path $SESSION_ROOTFS" curl --unix-socket $EDIT_FC_SOCK -i \ - -X PATCH 'http://localhost/vm' \ - -H 'Accept: application/json' \ - -H 'Content-Type: application/json' \ - -d '{ + -X PATCH 'http://localhost/vm' \ + -H 'Accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ "state": "Paused" }' curl --unix-socket $EDIT_FC_SOCK -i \ - -X PUT 'http://localhost/snapshot/create' \ - -H 'Accept: application/json' \ - -H 'Content-Type: application/json' \ - -d "{ + -X PUT 'http://localhost/snapshot/create' \ + -H 'Accept: application/json' \ + -H 'Content-Type: application/json' \ + -d "{ \"snapshot_type\": \"Full\", \"snapshot_path\": \"$BUILD_FC_SNAPFILE\", \"mem_file_path\": \"$BUILD_FC_MEMFILE\" @@ -124,10 +124,10 @@ function get_snapshot() { cp -pRd --reflink $SESSION_ROOTFS $BUILD_FC_ROOTFS curl --unix-socket $EDIT_FC_SOCK -i \ - -X PATCH 'http://localhost/vm' \ - -H 'Accept: application/json' \ - -H 'Content-Type: application/json' \ - -d '{ + -X PATCH 'http://localhost/vm' \ + -H 'Accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ "state": "Resumed" }' fi @@ -155,7 +155,6 @@ get_snapshot mv_env_files del_build_dir - touch ${ALLOC_DIR}/main-done echo "==== Output ===========================================================================================" diff --git a/packages/fc-env/use-prebuilt-env.sh b/packages/fc-env/use-prebuilt-env.sh index 327dedb6e5..d0f83749ff 100755 --- a/packages/fc-env/use-prebuilt-env.sh +++ b/packages/fc-env/use-prebuilt-env.sh @@ -40,7 +40,7 @@ if [ -z "$API_KEY" ]; then exit 1 fi -API_URL="https://ondevbook.com" +API_URL="https://api.e2b.dev" ENVS_ENDPOINT="${API_URL}/envs/${CODE_SNIPPET_ID}/state?api_key=$API_KEY" echo "==== Args ===========================================================================================" @@ -78,9 +78,9 @@ function mk_template_files() { if [ -f "$TEMPLATE_BUILD_ID" ]; then echo "$TEMPLATE_BUILD_ID exists - the template environemnt was rebuilt in this directory so we must copy the current build_id." - echo -n "${TEMPLATE}" > ${FINAL_TEMPLATE_ID_FILE} + echo -n "${TEMPLATE}" >${FINAL_TEMPLATE_ID_FILE} cp $TEMPLATE_BUILD_ID $FINAL_TEMPLATE_BUILD_ID_FILE - else + else echo "$TEMPLATE_BUILD_ID does not exist - the template environment uses build_id from its own template - we copy files pointing to its own template." cp $TEMPLATE_TEMPLATE_ID_FILE $FINAL_TEMPLATE_ID_FILE @@ -110,14 +110,12 @@ curl $ENVS_ENDPOINT \ \"state\": \"Building\" }" - mkdirs link_env_files mk_template_files touch ${ALLOC_DIR}/main-done - echo "==== Output ===========================================================================================" echo "| Code snippet ID: $CODE_SNIPPET_ID" echo "| Rootfs: $FINAL_FC_ROOTFS" diff --git a/packages/firecracker-task-driver/Makefile b/packages/firecracker-task-driver/Makefile index 972affb27e..fb09b1eb39 100644 --- a/packages/firecracker-task-driver/Makefile +++ b/packages/firecracker-task-driver/Makefile @@ -4,10 +4,10 @@ upload: ./upload.sh build: - CGO_ENABLED=0 go build -o bin/ . + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/ . set-project-gcloud: - gcloud config set project devbookhq + gcloud config set project e2b-prod generate: cd internal/client && swagger generate client -f firecracker.yml -A firecracker diff --git a/packages/firecracker-task-driver/upload.sh b/packages/firecracker-task-driver/upload.sh index 1d1ee32c63..5a822bf7fc 100755 --- a/packages/firecracker-task-driver/upload.sh +++ b/packages/firecracker-task-driver/upload.sh @@ -3,4 +3,4 @@ set -euo pipefail gsutil -h "Cache-Control:no-cache, max-age=0" \ - cp bin/firecracker-task-driver gs://devbook-environment-pipeline/firecracker-task-driver + cp bin/firecracker-task-driver gs://e2b-fc-env-pipeline/firecracker-task-driver diff --git a/packages/telemetry/logs-collector.hcl b/packages/telemetry/logs-collector.hcl index 3ce3e53562..fc6ffc32b8 100644 --- a/packages/telemetry/logs-collector.hcl +++ b/packages/telemetry/logs-collector.hcl @@ -18,7 +18,7 @@ variable "logs_port_name" { type = string } -variable "logtail_api_key" { +variable "betterstack_logs_api_key" { type = string } @@ -114,7 +114,7 @@ inputs = [ "logtail_transform_Ng2hNptjHFG5TLHYJnMKdrAY" ] uri = "https://in.logtail.com/" encoding.codec = "json" auth.strategy = "bearer" -auth.token = "${var.logtail_api_key}" +auth.token = "${var.betterstack_logs_api_key}" EOH } } diff --git a/packages/telemetry/main.tf b/packages/telemetry/main.tf index 71db7e77af..802c46b9da 100644 --- a/packages/telemetry/main.tf +++ b/packages/telemetry/main.tf @@ -16,12 +16,12 @@ resource "nomad_job" "logs-collector" { hcl2 { enabled = true vars = { - logtail_api_key = var.logtail_api_key - logs_port_number = var.logs_proxy_port.port - logs_health_port_number = var.logs_health_proxy_port.port - logs_health_path = var.logs_health_proxy_port.health_path - logs_port_name = var.logs_proxy_port.name - gcp_zone = var.gcp_zone + betterstack_logs_api_key = var.betterstack_logs_api_key + logs_port_number = var.logs_proxy_port.port + logs_health_port_number = var.logs_health_proxy_port.port + logs_health_path = var.logs_health_proxy_port.health_path + logs_port_name = var.logs_proxy_port.name + gcp_zone = var.gcp_zone } } } diff --git a/packages/telemetry/variables.tf b/packages/telemetry/variables.tf index a3d929cf4d..b103f7dec4 100644 --- a/packages/telemetry/variables.tf +++ b/packages/telemetry/variables.tf @@ -6,7 +6,7 @@ variable "lightstep_api_key" { type = string } -variable "logtail_api_key" { +variable "betterstack_logs_api_key" { type = string } diff --git a/variables.tf b/variables.tf index 6b3abfe1e0..d789e91bbe 100644 --- a/variables.tf +++ b/variables.tf @@ -1,7 +1,7 @@ variable "gcp_project_id" { description = "The project to deploy the cluster in" type = string - default = "devbookhq" + default = "e2b-prod" } variable "gcp_region" { @@ -22,7 +22,7 @@ variable "server_cluster_size" { variable "server_machine_type" { type = string - default = "n1-standard-4" + default = "n1-standard-1" } variable "client_cluster_size" { @@ -32,7 +32,7 @@ variable "client_cluster_size" { variable "client_machine_type" { type = string - default = "n1-standard-16" + default = "n1-standard-1" } variable "client_proxy_health_port" {