diff --git a/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl b/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl index f37d681cc9..b01ff50763 100644 --- a/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl +++ b/iac/modules/job-dashboard-api/jobs/dashboard-api.hcl @@ -80,6 +80,9 @@ job "dashboard-api" { AUTH_DB_READ_REPLICA_CONNECTION_STRING = "${auth_db_read_replica_connection_string}" CLICKHOUSE_CONNECTION_STRING = "${clickhouse_connection_string}" SUPABASE_JWT_SECRETS = "${supabase_jwt_secrets}" + REDIS_URL = "${redis_url}" + REDIS_CLUSTER_URL = "${redis_cluster_url}" + REDIS_TLS_CA_BASE64 = "${redis_tls_ca_base64}" OTEL_COLLECTOR_GRPC_ENDPOINT = "${otel_collector_grpc_endpoint}" LOGS_COLLECTOR_ADDRESS = "${logs_collector_address}" } diff --git a/iac/modules/job-dashboard-api/main.tf b/iac/modules/job-dashboard-api/main.tf index d1c455e3d1..c8fccdf69e 100644 --- a/iac/modules/job-dashboard-api/main.tf +++ b/iac/modules/job-dashboard-api/main.tf @@ -15,6 +15,9 @@ resource "nomad_job" "dashboard_api" { auth_db_read_replica_connection_string = var.auth_db_read_replica_connection_string clickhouse_connection_string = var.clickhouse_connection_string supabase_jwt_secrets = var.supabase_jwt_secrets + redis_url = var.redis_url + redis_cluster_url = var.redis_cluster_url + redis_tls_ca_base64 = var.redis_tls_ca_base64 subdomain = "dashboard-api" diff --git a/iac/modules/job-dashboard-api/variables.tf b/iac/modules/job-dashboard-api/variables.tf index 8b77e1c9a6..ae42e8c5c8 100644 --- a/iac/modules/job-dashboard-api/variables.tf +++ b/iac/modules/job-dashboard-api/variables.tf @@ -49,6 +49,22 @@ variable "otel_collector_grpc_port" { default = 4317 } +variable "redis_url" { + type = string + sensitive = true +} + +variable "redis_cluster_url" { + type = string + sensitive = true +} + +variable "redis_tls_ca_base64" { + type = string + sensitive = true + default = "" +} + variable "logs_proxy_port" { type = object({ name = string diff --git a/iac/provider-gcp/nomad/main.tf b/iac/provider-gcp/nomad/main.tf index f7889e2e35..ea6632643a 100644 --- a/iac/provider-gcp/nomad/main.tf +++ b/iac/provider-gcp/nomad/main.tf @@ -135,6 +135,9 @@ module "dashboard_api" { auth_db_read_replica_connection_string = trimspace(data.google_secret_manager_secret_version.postgres_read_replica_connection_string.secret_data) clickhouse_connection_string = local.clickhouse_connection_string supabase_jwt_secrets = trimspace(data.google_secret_manager_secret_version.supabase_jwt_secrets.secret_data) + redis_url = local.redis_url + redis_cluster_url = local.redis_cluster_url + redis_tls_ca_base64 = trimspace(data.google_secret_manager_secret_version.redis_tls_ca_base64.secret_data) otel_collector_grpc_port = var.otel_collector_grpc_port logs_proxy_port = var.logs_proxy_port diff --git a/packages/api/internal/handlers/store.go b/packages/api/internal/handlers/store.go index 18f3874554..a219321e5f 100644 --- a/packages/api/internal/handlers/store.go +++ b/packages/api/internal/handlers/store.go @@ -150,7 +150,7 @@ func NewAPIStore(ctx context.Context, tel *telemetry.Client, redisClient redis.U logger.L().Fatal(ctx, "Initializing Orchestrator client", zap.Error(err)) } - authCache := sharedauth.NewAuthCache[*types.Team]() + authCache := sharedauth.NewAuthCache[*types.Team](redisClient) authStore := sharedauth.NewAuthStore(authDB) authService := sharedauth.NewAuthService[*types.Team](authStore, authCache, config.SupabaseJWTSecrets) templateCache := templatecache.NewTemplateCache(sqlcDB, redisClient) diff --git a/packages/auth/go.mod b/packages/auth/go.mod index 27782f8e62..0e41b3298c 100644 --- a/packages/auth/go.mod +++ b/packages/auth/go.mod @@ -14,6 +14,7 @@ require ( github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 github.com/oapi-codegen/gin-middleware v1.0.2 + github.com/redis/go-redis/v9 v9.17.3 github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.41.0 go.uber.org/zap v1.27.1 @@ -94,7 +95,6 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect - github.com/redis/go-redis/v9 v9.17.3 // indirect github.com/shirou/gopsutil/v4 v4.25.9 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/testcontainers/testcontainers-go v0.40.0 // indirect diff --git a/packages/auth/pkg/auth/cache.go b/packages/auth/pkg/auth/cache.go index a7bf63c7ec..3f83fb6816 100644 --- a/packages/auth/pkg/auth/cache.go +++ b/packages/auth/pkg/auth/cache.go @@ -4,6 +4,8 @@ import ( "context" "time" + "github.com/redis/go-redis/v9" + "github.com/e2b-dev/infra/packages/shared/pkg/cache" ) @@ -11,25 +13,25 @@ const ( authInfoExpiration = 5 * time.Minute refreshInterval = 1 * time.Minute refreshTimeout = 30 * time.Second - callbackTimeout = 30 * time.Second + + authCacheRedisPrefix = "auth:team" ) -// AuthCache is a generic TTL cache for authentication data (teams, users, etc.). +// AuthCache is a Redis-backed TTL cache for authentication data (teams, users, etc.). type AuthCache[T any] struct { - cache *cache.MemoryCache[T] + cache *cache.RedisCache[T] } -// NewAuthCache creates a new AuthCache with default TTL and refresh settings. -func NewAuthCache[T any]() *AuthCache[T] { - config := cache.Config[T]{ - TTL: authInfoExpiration, - RefreshInterval: refreshInterval, - RefreshTimeout: refreshTimeout, - CallbackTimeout: callbackTimeout, - } - +// NewAuthCache creates a new Redis-backed AuthCache with default TTL and refresh settings. +func NewAuthCache[T any](redisClient redis.UniversalClient) *AuthCache[T] { return &AuthCache[T]{ - cache: cache.NewMemoryCache(config), + cache: cache.NewRedisCache(cache.RedisConfig[T]{ + RedisClient: redisClient, + TTL: authInfoExpiration, + RefreshInterval: refreshInterval, + RefreshTimeout: refreshTimeout, + RedisPrefix: authCacheRedisPrefix, + }), } } @@ -39,11 +41,11 @@ func (c *AuthCache[T]) GetOrSet(ctx context.Context, key string, dataCallback fu } // Invalidate removes a single entry from the cache by key. -func (c *AuthCache[T]) Invalidate(key string) { - c.cache.Delete(key) +func (c *AuthCache[T]) Invalidate(ctx context.Context, key string) { + c.cache.Delete(ctx, key) } -// Close stops the cache's background refresh goroutines. +// Close is a no-op for the Redis-backed cache (no background goroutines). func (c *AuthCache[T]) Close(ctx context.Context) error { return c.cache.Close(ctx) } diff --git a/packages/auth/pkg/auth/service.go b/packages/auth/pkg/auth/service.go index 13133420c4..6853ef1b9c 100644 --- a/packages/auth/pkg/auth/service.go +++ b/packages/auth/pkg/auth/service.go @@ -200,8 +200,8 @@ func (s *AuthService[T]) ValidateSupabaseTeam(ctx context.Context, ginCtx *gin.C // InvalidateTeamMemberCache removes the cached auth entry for a specific user-team pair. // This should be called when team membership changes (member added or removed). -func (s *AuthService[T]) InvalidateTeamMemberCache(userID uuid.UUID, teamID string) { - s.teamCache.Invalidate(supabaseTeamCacheKey(userID, teamID)) +func (s *AuthService[T]) InvalidateTeamMemberCache(ctx context.Context, userID uuid.UUID, teamID string) { + s.teamCache.Invalidate(ctx, supabaseTeamCacheKey(userID, teamID)) } // InvalidateTeamCache queries the team's API key hashes and removes their cached entries. @@ -212,7 +212,7 @@ func (s *AuthService[T]) InvalidateTeamCache(ctx context.Context, teamID uuid.UU } for _, hash := range hashes { - s.teamCache.Invalidate(hash) + s.teamCache.Invalidate(ctx, hash) } return nil diff --git a/packages/dashboard-api/go.mod b/packages/dashboard-api/go.mod index b4989c137a..f42d39bdba 100644 --- a/packages/dashboard-api/go.mod +++ b/packages/dashboard-api/go.mod @@ -111,6 +111,8 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/pressly/goose/v3 v3.26.0 // indirect + github.com/redis/go-redis/extra/rediscmd/v9 v9.17.3 // indirect + github.com/redis/go-redis/extra/redisotel/v9 v9.17.3 // indirect github.com/redis/go-redis/v9 v9.17.3 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect diff --git a/packages/dashboard-api/go.sum b/packages/dashboard-api/go.sum index 8bc4994cc9..b83791e2c0 100644 --- a/packages/dashboard-api/go.sum +++ b/packages/dashboard-api/go.sum @@ -241,6 +241,10 @@ github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/pressly/goose/v3 v3.26.0 h1:KJakav68jdH0WDvoAcj8+n61WqOIaPGgH0bJWS6jpmM= github.com/pressly/goose/v3 v3.26.0/go.mod h1:4hC1KrritdCxtuFsqgs1R4AU5bWtTAf+cnWvfhf2DNY= +github.com/redis/go-redis/extra/rediscmd/v9 v9.17.3 h1:v9RNP5ynWkruvzscrIoDyyv20c9YeyVn12L9nYnaexw= +github.com/redis/go-redis/extra/rediscmd/v9 v9.17.3/go.mod h1:gdthSemCkR3WxTmzV2XxYIxClunkUJZAhL0zPHaB0Ww= +github.com/redis/go-redis/extra/redisotel/v9 v9.17.3 h1:bF0e3fV7PL0knd1UHDtMud8wA7CZt3RSWtyTMhpnWd8= +github.com/redis/go-redis/extra/redisotel/v9 v9.17.3/go.mod h1:gR39sPK/dJZlqgIA9Nm4JFHcQJPyhsISBLj708nrD4w= github.com/redis/go-redis/v9 v9.17.3 h1:fN29NdNrE17KttK5Ndf20buqfDZwGNgoUr9qjl1DQx4= github.com/redis/go-redis/v9 v9.17.3/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= diff --git a/packages/dashboard-api/internal/cfg/model.go b/packages/dashboard-api/internal/cfg/model.go index f0d9ad10a1..669825689b 100644 --- a/packages/dashboard-api/internal/cfg/model.go +++ b/packages/dashboard-api/internal/cfg/model.go @@ -1,6 +1,8 @@ package cfg import ( + "fmt" + "github.com/caarlos0/env/v11" ) @@ -12,6 +14,10 @@ type Config struct { AuthDBConnectionString string `env:"AUTH_DB_CONNECTION_STRING"` AuthDBReadReplicaConnectionString string `env:"AUTH_DB_READ_REPLICA_CONNECTION_STRING"` + + RedisURL string `env:"REDIS_URL"` + RedisClusterURL string `env:"REDIS_CLUSTER_URL"` + RedisTLSCABase64 string `env:"REDIS_TLS_CA_BASE64"` } func Parse() (Config, error) { @@ -22,5 +28,9 @@ func Parse() (Config, error) { config.AuthDBConnectionString = config.PostgresConnectionString } + if err == nil && config.RedisURL == "" && config.RedisClusterURL == "" { + err = fmt.Errorf("at least one of REDIS_URL or REDIS_CLUSTER_URL must be set") + } + return config, err } diff --git a/packages/dashboard-api/internal/handlers/team_members.go b/packages/dashboard-api/internal/handlers/team_members.go index c8890ca416..6521dd6f4b 100644 --- a/packages/dashboard-api/internal/handlers/team_members.go +++ b/packages/dashboard-api/internal/handlers/team_members.go @@ -107,7 +107,7 @@ func (s *APIStore) PostTeamsTeamIDMembers(c *gin.Context, teamID api.TeamID) { return } - s.authService.InvalidateTeamMemberCache(user.ID, teamInfo.Team.ID.String()) + s.authService.InvalidateTeamMemberCache(ctx, user.ID, teamInfo.Team.ID.String()) c.Status(http.StatusCreated) } @@ -189,7 +189,7 @@ func (s *APIStore) DeleteTeamsTeamIDMembersUserId(c *gin.Context, teamID api.Tea return } - s.authService.InvalidateTeamMemberCache(userId, teamInfo.Team.ID.String()) + s.authService.InvalidateTeamMemberCache(ctx, userId, teamInfo.Team.ID.String()) c.Status(http.StatusNoContent) } diff --git a/packages/dashboard-api/main.go b/packages/dashboard-api/main.go index 4b40634473..ad924fce4c 100644 --- a/packages/dashboard-api/main.go +++ b/packages/dashboard-api/main.go @@ -34,6 +34,7 @@ import ( authdb "github.com/e2b-dev/infra/packages/db/pkg/auth" "github.com/e2b-dev/infra/packages/db/pkg/pool" e2benv "github.com/e2b-dev/infra/packages/shared/pkg/env" + "github.com/e2b-dev/infra/packages/shared/pkg/factories" "github.com/e2b-dev/infra/packages/shared/pkg/logger" sharedmiddleware "github.com/e2b-dev/infra/packages/shared/pkg/middleware" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" @@ -140,7 +141,21 @@ func run() int { defer clickhouseClient.Close(ctx) } - authCache := sharedauth.NewAuthCache[*types.Team]() + redisClient, err := factories.NewRedisClient(ctx, factories.RedisConfig{ + RedisURL: config.RedisURL, + RedisClusterURL: config.RedisClusterURL, + RedisTLSCABase64: config.RedisTLSCABase64, + }) + if err != nil { + l.Fatal(ctx, "Initializing Redis client", zap.Error(err)) + } + defer func() { + if err := factories.CloseCleanly(redisClient); err != nil { + l.Error(ctx, "Failed to close Redis client", zap.Error(err)) + } + }() + + authCache := sharedauth.NewAuthCache[*types.Team](redisClient) authStore := sharedauth.NewAuthStore(authDB) authService := sharedauth.NewAuthService[*types.Team](authStore, authCache, config.SupabaseJWTSecrets) defer authService.Close(ctx)