From 770046f91eb3f90401e861f8c10409e9d8470d90 Mon Sep 17 00:00:00 2001 From: Trey Date: Fri, 13 Feb 2026 11:08:54 -0800 Subject: [PATCH 1/3] Add Redis Sentinel storage documentation Add comprehensive documentation for configuring Redis Sentinel as the storage backend for the embedded authorization server, enabling horizontal scaling across multiple auth server replicas. New files: - docs/redis-storage.md: User-facing configuration guide with CRD examples, Spotahome Redis Operator deployment steps, data model reference, troubleshooting, and configuration reference tables - docs/arch/11-auth-server-storage.md: Architecture documentation covering storage interface design, memory and Redis backends, multi-tenancy via key prefixes, and atomic Lua script operations - examples/operator/redis-storage/: Example manifests for Redis Failover, Sentinel Service, credentials Secret, and MCPExternalAuthConfig with Redis storage Updated architecture docs (README, overview, operator) to reference the new storage documentation. Closes #3790 --- docs/arch/00-overview.md | 1 + docs/arch/09-operator-architecture.md | 2 +- docs/arch/11-auth-server-storage.md | 194 ++++++++ docs/arch/README.md | 12 +- docs/redis-storage.md | 420 ++++++++++++++++++ .../mcpexternalauthconfig-redis-storage.yaml | 48 ++ .../redis-storage/redis-credentials.yaml | 18 + .../redis-storage/redis-failover.yaml | 70 +++ .../redis-storage/sentinel-service.yaml | 121 +++++ 9 files changed, 884 insertions(+), 2 deletions(-) create mode 100644 docs/arch/11-auth-server-storage.md create mode 100644 docs/redis-storage.md create mode 100644 examples/operator/redis-storage/mcpexternalauthconfig-redis-storage.yaml create mode 100644 examples/operator/redis-storage/redis-credentials.yaml create mode 100644 examples/operator/redis-storage/redis-failover.yaml create mode 100644 examples/operator/redis-storage/sentinel-service.yaml diff --git a/docs/arch/00-overview.md b/docs/arch/00-overview.md index d07b2567f0..f4341ed3b2 100644 --- a/docs/arch/00-overview.md +++ b/docs/arch/00-overview.md @@ -260,6 +260,7 @@ These are automatically converted to container images at runtime. - [Workloads Lifecycle](08-workloads-lifecycle.md) - Workload management - [Operator Architecture](09-operator-architecture.md) - Kubernetes operator design - [Virtual MCP Server Architecture](10-virtual-mcp-architecture.md) - MCP Gateway and aggregation +- [Auth Server Storage](11-auth-server-storage.md) - Memory and Redis Sentinel storage backends ## Getting Started diff --git a/docs/arch/09-operator-architecture.md b/docs/arch/09-operator-architecture.md index 94f06c78af..23155de4a2 100644 --- a/docs/arch/09-operator-architecture.md +++ b/docs/arch/09-operator-architecture.md @@ -172,7 +172,7 @@ Manages external authentication configurations that can be shared across multipl **Implementation**: `cmd/thv-operator/api/v1alpha1/mcpexternalauthconfig_types.go` -MCPExternalAuthConfig allows you to define reusable OIDC authentication configurations that can be referenced by multiple MCPServer resources. This is useful for sharing authentication settings across servers. +MCPExternalAuthConfig allows you to define reusable OIDC authentication configurations that can be referenced by multiple MCPServer resources. This is useful for sharing authentication settings across servers. When using the embedded auth server type, the `storage` field supports configuring Redis Sentinel as a shared storage backend for horizontal scaling. See [Auth Server Storage](11-auth-server-storage.md) for details. **Referenced by MCPServer** using `oidcConfig.type: external`. diff --git a/docs/arch/11-auth-server-storage.md b/docs/arch/11-auth-server-storage.md new file mode 100644 index 0000000000..dc215a5f91 --- /dev/null +++ b/docs/arch/11-auth-server-storage.md @@ -0,0 +1,194 @@ +# Auth Server Storage Architecture + +The embedded authorization server uses a pluggable storage backend to persist OAuth 2.0 state. This document describes the storage architecture, the available backends, and the Redis Sentinel implementation. + +## Overview + +The auth server stores OAuth 2.0 protocol state including access tokens, refresh tokens, authorization codes, PKCE challenges, client registrations, user accounts, and upstream IDP tokens. Two storage backends are available: + +1. **Memory** (default): In-process storage with mutex-based concurrency. Suitable for single-instance deployments. +2. **Redis Sentinel**: Shared storage using Redis with Sentinel for high availability. Required for horizontal scaling across multiple auth server replicas. + +```mermaid +graph TB + subgraph "Auth Server Replicas" + AS1[Auth Server 1] + AS2[Auth Server 2] + AS3[Auth Server N] + end + + subgraph "Storage Backend" + direction TB + Memory[In-Memory Storage
Single instance only] + Redis[Redis Sentinel
Shared state] + end + + AS1 -.->|single instance| Memory + AS1 -->|distributed| Redis + AS2 -->|distributed| Redis + AS3 -->|distributed| Redis + + subgraph "Redis Sentinel Cluster" + S1[Sentinel 1] + S2[Sentinel 2] + S3[Sentinel 3] + RM[Redis Master] + RR1[Redis Replica] + RR2[Redis Replica] + end + + Redis --> S1 + Redis --> S2 + Redis --> S3 + S1 -.->|monitors| RM + S2 -.->|monitors| RM + S3 -.->|monitors| RM + RM -->|replicates| RR1 + RM -->|replicates| RR2 + + style Memory fill:#fff3e0 + style Redis fill:#e1f5fe + style RM fill:#ffb74d +``` + +## Storage Interface + +The storage layer implements multiple interfaces from the [fosite](https://github.com/ory/fosite) OAuth 2.0 framework, plus ToolHive-specific extensions: + +**Fosite interfaces:** +- `oauth2.AuthorizeCodeStorage` — Authorization code grant +- `oauth2.AccessTokenStorage` — Access token persistence +- `oauth2.RefreshTokenStorage` — Refresh token with rotation +- `oauth2.TokenRevocationStorage` — Token revocation (RFC 7009) +- `pkce.PKCERequestStorage` — PKCE challenge/verifier (RFC 7636) + +**ToolHive extensions:** +- `ClientRegistry` — Dynamic client registration (RFC 7591) +- `UpstreamTokenStorage` — Upstream IDP token caching with user binding +- `PendingAuthorizationStorage` — In-flight authorization tracking +- `UserStorage` — Internal user accounts and provider identity linking + +**Implementation:** +- Interface definitions: `pkg/authserver/storage/types.go` +- Memory backend: `pkg/authserver/storage/memory.go` +- Redis backend: `pkg/authserver/storage/redis.go` + +## Memory Backend + +The in-memory backend uses Go maps protected by `sync.RWMutex` for thread safety. A background goroutine runs periodic cleanup of expired entries. + +**Characteristics:** +- Zero external dependencies +- State is lost on restart +- Cannot be shared across replicas +- Suitable for development and single-instance deployments + +**Implementation:** `pkg/authserver/storage/memory.go` + +## Redis Sentinel Backend + +The Redis backend stores all OAuth 2.0 state as JSON-serialized values in Redis, using the Sentinel protocol for automatic master discovery and failover. + +### Connection Architecture + +The client connects to Redis through Sentinel using `redis.NewFailoverClient()` from the `go-redis` library. Sentinel handles: +- Master discovery: Finding the current master node +- Automatic failover: Detecting master failure and promoting a replica +- Configuration notification: Updating clients when the master changes + +### Multi-Tenancy + +Each auth server instance has a unique key prefix derived from its Kubernetes namespace and name: + +``` +thv:auth:{namespace:name}: +``` + +The `{namespace:name}` portion is a Redis hash tag. Although ToolHive only supports Sentinel deployments, the hash tag format ensures keys remain co-located in the same hash slot if the deployment were ever migrated to Redis Cluster. In Sentinel mode, hash tags have no functional effect but impose no overhead. + +**Implementation:** `pkg/authserver/storage/redis_keys.go` + +### Key Design + +Keys follow the pattern `{prefix}{type}:{id}`: + +``` +thv:auth:{default:my-server}:access:abc123 +thv:auth:{default:my-server}:refresh:def456 +thv:auth:{default:my-server}:user:user-uuid +``` + +Secondary indexes use Redis Sets to enable reverse lookups: + +``` +thv:auth:{default:my-server}:reqid:access:{request-id} → {sig1, sig2} +thv:auth:{default:my-server}:user:upstream:{user-id} → {session1, session2} +``` + +### Consistency Model + +The implementation uses different strategies based on consistency requirements: + +- **Lua scripts** for strict atomicity: upstream token storage with user reverse-index cleanup, last-used timestamp updates +- **Pipelines** (`MULTI`/`EXEC`) for batched operations: authorization code invalidation, token session creation with secondary index updates +- **Individual commands** with best-effort cleanup: token revocation, refresh token rotation — partial failures are safe since orphaned keys expire via TTL + +### Serialization + +All values are stored as JSON. The implementation uses defensive copies on read and write to prevent caller mutations from affecting stored data. + +### TTL Management + +Redis TTL (`SETEX`) is used for all time-bounded data. TTL values are derived from OAuth 2.0 token lifetimes: + +| Data Type | Default TTL | +|---|---| +| Access tokens | 1 hour | +| Refresh tokens | 30 days | +| Authorization codes | 10 minutes | +| PKCE requests | 10 minutes | +| Invalidated codes | 30 minutes | +| Public clients (DCR) | 30 days | +| Users / Providers | No expiry | + +## Configuration + +### CRD Configuration + +In Kubernetes, storage is configured via the `MCPExternalAuthConfig` CRD: + +``` +MCPExternalAuthConfig + └── spec.embeddedAuthServer.storage + ├── type: "memory" | "redis" + └── redis + ├── sentinelConfig + │ ├── masterName + │ ├── sentinelAddrs[] (or sentinelService) + │ └── db + ├── aclUserConfig + │ ├── usernameSecretRef + │ └── passwordSecretRef + └── timeouts (dial, read, write) +``` + +**Implementation:** `cmd/thv-operator/api/v1alpha1/mcpexternalauthconfig_types.go` + +### RunConfig Serialization + +When passing configuration across process boundaries (operator → proxy-runner), the CRD configuration is converted to `RunConfig` format where Secret references become environment variable references. + +**Implementation:** `pkg/authserver/storage/config.go` + +## Security Considerations + +- **ACL authentication only**: Redis ACL users (Redis 6+) provide fine-grained access control. Legacy `requirepass` authentication is not supported. +- **Key prefix isolation**: Each auth server is restricted to its own key prefix via Redis ACL rules (`~thv:auth:*`). +- **Credential handling**: In Kubernetes, credentials are stored in Secrets and injected as environment variables. They are never written to disk or logged. +- **No TLS currently**: TLS/mTLS for Redis connections is not yet supported and is planned as a future enhancement. + +## Related Documentation + +- [Redis Storage Configuration Guide](../redis-storage.md) — User-facing setup guide +- [Operator Architecture](09-operator-architecture.md) — CRD and controller design +- [Core Concepts](02-core-concepts.md) — Platform terminology diff --git a/docs/arch/README.md b/docs/arch/README.md index d2073a74c7..d44860860e 100644 --- a/docs/arch/README.md +++ b/docs/arch/README.md @@ -76,6 +76,13 @@ Welcome to the ToolHive architecture documentation. This directory contains comp - Two-boundary authentication model - Composite tool workflows +12. **[Auth Server Storage Architecture](11-auth-server-storage.md)** + - Storage interface design (fosite + ToolHive extensions) + - Memory and Redis Sentinel backends + - Multi-tenancy via key prefixes + - Atomic operations with Lua scripts + - Configuration and security model + ### Existing Documentation For middleware architecture, see: **[docs/middleware.md](../middleware.md)** @@ -118,6 +125,7 @@ graph TB Workloads[08: Workloads Lifecycle
Deploy, stop, restart, delete] Operator[09: Kubernetes Operator
CRDs & reconciliation] vMCP[10: Virtual MCP
Aggregation & Gateway] + AuthStorage[11: Auth Server Storage
Memory & Redis backends] end %% Navigation paths @@ -144,6 +152,7 @@ graph TB Workloads --> Operator vMCP --> Operator + AuthStorage --> Operator %% Styling style Overview fill:#e1f5fe,stroke:#01579b,stroke-width:3px @@ -158,6 +167,7 @@ graph TB style Workloads fill:#e0f2f1,stroke:#004d40,stroke-width:2px style Operator fill:#e0f2f1,stroke:#004d40,stroke-width:2px style vMCP fill:#e0f2f1,stroke:#004d40,stroke-width:2px + style AuthStorage fill:#e0f2f1,stroke:#004d40,stroke-width:2px ``` **Color Legend:** @@ -383,5 +393,5 @@ Links to related docs --- **Version**: 0.1.0 (Initial architecture documentation) -**Last Updated**: 2025-10-13 +**Last Updated**: 2026-02-13 **Maintainers**: ToolHive Core Team diff --git a/docs/redis-storage.md b/docs/redis-storage.md new file mode 100644 index 0000000000..35aaea009f --- /dev/null +++ b/docs/redis-storage.md @@ -0,0 +1,420 @@ +# Redis Sentinel Storage for Auth Server + +This guide explains how to configure Redis Sentinel as the storage backend for ToolHive's embedded authorization server, enabling horizontal scaling across multiple auth server replicas. + +## Overview + +By default, ToolHive's embedded auth server uses in-memory storage. This works well for single-instance deployments but does not support horizontal scaling since each replica has its own isolated state. Redis Sentinel provides a shared, highly available storage backend that enables multiple auth server replicas to share OAuth 2.0 state (tokens, authorization codes, clients, and user data). + +**Key design decisions:** + +- **Sentinel-only**: Only Redis Sentinel deployments are supported (not standalone or cluster mode). Sentinel provides automatic failover and high availability without the complexity of Redis Cluster. +- **ACL user authentication**: Only Redis ACL user authentication is supported. This is the modern Redis authentication mechanism (Redis 6+) that provides fine-grained access control. +- **Multi-tenancy via key prefixes**: Each auth server instance uses a unique key prefix (`thv:auth:{namespace:name}:`) to isolate its data, allowing multiple auth servers to share the same Redis deployment. + +## Prerequisites + +- A running Redis Sentinel deployment (Redis 6+ for ACL support) +- Redis ACL user configured with appropriate permissions +- For Kubernetes: Secrets containing Redis credentials + +## Configuration + +### Kubernetes (MCPExternalAuthConfig CRD) + +When using the ToolHive operator, Redis storage is configured through the `storage` field in the embedded auth server section of `MCPExternalAuthConfig`. + +```yaml +apiVersion: toolhive.stacklok.dev/v1alpha1 +kind: MCPExternalAuthConfig +metadata: + name: my-auth-config + namespace: default +spec: + type: embeddedAuthServer + embeddedAuthServer: + # ... other auth server config ... + + storage: + type: redis + redis: + sentinelConfig: + masterName: mymaster + # Option 1: Direct Sentinel addresses + sentinelAddrs: + - "redis-sentinel-0.redis-sentinel:26379" + - "redis-sentinel-1.redis-sentinel:26379" + - "redis-sentinel-2.redis-sentinel:26379" + db: 0 + + aclUserConfig: + usernameSecretRef: + name: redis-credentials + key: username + passwordSecretRef: + name: redis-credentials + key: password + + # Optional timeouts (shown with defaults) + dialTimeout: "5s" + readTimeout: "3s" + writeTimeout: "3s" +``` + +#### Sentinel Service Discovery + +Instead of listing Sentinel addresses directly, you can reference a Kubernetes Service. The operator resolves the Service's Endpoints to discover Sentinel instances automatically. + +```yaml +storage: + type: redis + redis: + sentinelConfig: + masterName: mymaster + # Option 2: Kubernetes Service discovery + sentinelService: + name: rfs-redis-sentinel + namespace: redis # defaults to same namespace if omitted + port: 26379 # defaults to 26379 if omitted + db: 0 + + aclUserConfig: + usernameSecretRef: + name: redis-credentials + key: username + passwordSecretRef: + name: redis-credentials + key: password +``` + +> **Note:** `sentinelAddrs` and `sentinelService` are mutually exclusive. Specify one or the other. + +#### Redis Credentials Secret + +Create a Kubernetes Secret containing the Redis ACL username and password: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: redis-credentials + namespace: default +type: Opaque +stringData: + username: toolhive-auth + password: "" +``` + +### RunConfig (Process Boundary Configuration) + +When the auth server configuration is serialized for passing across process boundaries (e.g., from operator to proxy-runner), it uses the `RunConfig` format: + +```json +{ + "type": "redis", + "redisConfig": { + "sentinelConfig": { + "masterName": "mymaster", + "sentinelAddrs": [ + "redis-sentinel-0:26379", + "redis-sentinel-1:26379", + "redis-sentinel-2:26379" + ], + "db": 0 + }, + "authType": "aclUser", + "aclUserConfig": { + "usernameEnvVar": "TOOLHIVE_AS_REDIS_USERNAME", + "passwordEnvVar": "TOOLHIVE_AS_REDIS_PASSWORD" + }, + "keyPrefix": "thv:auth:{default:my-auth-config}:", + "dialTimeout": "5s", + "readTimeout": "3s", + "writeTimeout": "3s" + } +} +``` + +In RunConfig format, credentials are referenced via environment variables rather than Kubernetes Secrets. The operator handles the translation from Secret references to environment variables when constructing the proxy-runner pod. + +## Deploying Redis with the Spotahome Redis Operator + +The [Spotahome Redis Operator](https://github.com/spotahome/redis-operator) provides a Kubernetes-native way to deploy and manage Redis Sentinel clusters. This section walks through deploying a Redis Sentinel cluster suitable for ToolHive's auth server storage. + +### Step 1: Install the Redis Operator + +```bash +# Using Helm +helm repo add redis-operator https://spotahome.github.io/redis-operator +helm repo update + +helm install redis-operator redis-operator/redis-operator \ + --namespace redis-operator \ + --create-namespace +``` + +### Step 2: Create the Redis Failover Resource + +The `RedisFailover` CRD deploys a Redis master-replica set with Sentinel monitoring: + +```yaml +apiVersion: databases.spotahome.com/v1 +kind: RedisFailover +metadata: + name: redis + namespace: redis +spec: + sentinel: + replicas: 3 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + redis: + replicas: 3 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + customConfig: + - "aclfile /data/users.acl" + storage: + persistentVolumeClaim: + metadata: + name: redis-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +``` + +### Step 3: Configure Redis ACL Users + +Create a ConfigMap or init container to provision the ACL file. The ACL user needs permissions on the key prefix used by ToolHive: + +``` +# /data/users.acl +user toolhive-auth on > ~thv:auth:* &* +@all +``` + +This ACL entry: +- `on` — Enables the user +- `>` — Sets the password +- `~thv:auth:*` — Allows access to all keys with the `thv:auth:` prefix +- `&*` — Allows access to all Pub/Sub channels (required for Sentinel) +- `+@all` — Allows all commands + +> **Security note:** In production, restrict the allowed commands to the minimum required set. The auth server uses `GET`, `SET`, `DEL`, `EXISTS`, `EXPIRE`, `SADD`, `SREM`, `SMEMBERS`, `EVAL`, `MULTI`, `EXEC`, and `PING`. + +### Step 4: Create the ToolHive Auth Config + +With the Redis Sentinel cluster running, configure ToolHive to use it: + +```yaml +# Redis credentials Secret +apiVersion: v1 +kind: Secret +metadata: + name: redis-credentials + namespace: default +type: Opaque +stringData: + username: toolhive-auth + password: "" +--- +# MCPExternalAuthConfig with Redis storage +apiVersion: toolhive.stacklok.dev/v1alpha1 +kind: MCPExternalAuthConfig +metadata: + name: my-auth-config + namespace: default +spec: + type: embeddedAuthServer + embeddedAuthServer: + issuer: "https://auth.example.com" + upstreamProviders: + - name: my-idp + type: oidc + oidcConfig: + issuerUrl: https://accounts.google.com + clientId: "my-client-id" + clientSecretRef: + name: idp-client-secret + key: client-secret + storage: + type: redis + redis: + sentinelConfig: + masterName: mymaster + sentinelService: + name: rfs-redis-sentinel + namespace: redis + aclUserConfig: + usernameSecretRef: + name: redis-credentials + key: username + passwordSecretRef: + name: redis-credentials + key: password +``` + +## Data Model + +### Key Schema + +All keys use the prefix `thv:auth:{namespace:name}:` where `{namespace:name}` is a Redis hash tag ensuring all keys for a single auth server land in the same hash slot. + +| Key Pattern | Purpose | TTL | +|---|---|---| +| `{prefix}access:{signature}` | Access token data | 1 hour (default) | +| `{prefix}refresh:{signature}` | Refresh token data | 30 days (default) | +| `{prefix}authcode:{code}` | Authorization code | 10 minutes | +| `{prefix}pkce:{signature}` | PKCE challenge data | 10 minutes | +| `{prefix}client:{client_id}` | OAuth client registration | 30 days (public) / none (confidential) | +| `{prefix}user:{user_id}` | User account | None | +| `{prefix}provider:{len}:{provider_id}:{subject}` | Provider identity linkage | None | +| `{prefix}upstream:{session_id}` | Upstream IDP tokens | Matches token lifetime | +| `{prefix}pending:{state}` | In-flight authorization | 10 minutes | +| `{prefix}invalidated:{code}` | Replay detection for auth codes | 30 minutes | +| `{prefix}jwt:{jti}` | Client assertion JWT replay prevention | Matches JWT `exp` | + +### Secondary Indexes + +Redis Sets are used as secondary indexes for efficient lookups: + +| Set Key Pattern | Purpose | +|---|---| +| `{prefix}reqid:access:{request_id}` | Request ID → access token signatures | +| `{prefix}reqid:refresh:{request_id}` | Request ID → refresh token signatures | +| `{prefix}user:upstream:{user_id}` | User → upstream token session IDs | +| `{prefix}user:providers:{user_id}` | User → provider identity keys | + +These indexes enable grant-wide operations like token revocation (finding all tokens for a request ID) and user-scoped queries (finding all upstream tokens for a user). + +### Atomicity and Consistency + +The storage implementation uses different strategies depending on the consistency requirements of each operation: + +- **Lua scripts** for strict atomicity: upstream token storage with user reverse-index cleanup, last-used timestamp updates +- **Pipelines** (`MULTI`/`EXEC`) for batched operations: authorization code invalidation, token session creation with secondary index updates +- **Individual commands** with best-effort cleanup: token revocation, refresh token rotation. These operations use `SMEMBERS` + individual `DEL` calls, meaning partial failures are possible but safe (orphaned keys expire via TTL) + +Secondary index cleanup is best-effort: stale entries may remain temporarily but are cleaned up on the next write or by TTL expiration. + +## Troubleshooting + +### Connection Failures + +**Symptom:** Auth server fails to start with Redis connection errors. + +**Checks:** +1. Verify Sentinel addresses are reachable from the auth server pod: + ```bash + kubectl exec -it -- nc -zv 26379 + ``` +2. Verify the master name matches the Sentinel configuration: + ```bash + redis-cli -h -p 26379 SENTINEL get-master-addr-by-name mymaster + ``` +3. Check that the ACL user credentials are correct: + ```bash + redis-cli -h -p 6379 --user toolhive-auth --pass PING + ``` + +### Authentication Errors + +**Symptom:** `WRONGPASS` or `NOAUTH` errors in logs. + +**Checks:** +1. Verify the Secret exists and contains the correct keys: + ```bash + kubectl get secret redis-credentials -o jsonpath='{.data.username}' | base64 -d + kubectl get secret redis-credentials -o jsonpath='{.data.password}' | base64 -d + ``` +2. Verify the ACL user exists on Redis: + ```bash + redis-cli -h -p 6379 ACL LIST + ``` + +### Key Permission Errors + +**Symptom:** `NOPERM` errors when accessing keys. + +**Checks:** +1. Verify the ACL user has the correct key pattern permissions: + ```bash + redis-cli -h -p 6379 ACL GETUSER toolhive-auth + ``` +2. Ensure the key pattern includes the `thv:auth:` prefix: + ``` + user toolhive-auth on > ~thv:auth:* &* +@all + ``` + +### Failover Issues + +**Symptom:** Requests fail during Redis master failover. + +**Notes:** +- The Redis client library handles Sentinel failover automatically. During a failover (typically a few seconds), requests may briefly fail and retry. +- Ensure at least 3 Sentinel instances for quorum-based failover. +- Monitor Sentinel logs for failover events: + ```bash + kubectl logs | grep "failover" + ``` + +## Configuration Reference + +### AuthServerStorageConfig (CRD) + +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `type` | `string` | No | `memory` | Storage backend type: `memory` or `redis` | +| `redis` | `RedisStorageConfig` | When type=redis | — | Redis configuration | + +### RedisStorageConfig (CRD) + +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `sentinelConfig` | `RedisSentinelConfig` | Yes | — | Sentinel connection settings | +| `aclUserConfig` | `RedisACLUserConfig` | Yes | — | ACL user credentials | +| `dialTimeout` | `string` | No | `5s` | Connection establishment timeout | +| `readTimeout` | `string` | No | `3s` | Socket read timeout | +| `writeTimeout` | `string` | No | `3s` | Socket write timeout | + +### RedisSentinelConfig (CRD) + +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `masterName` | `string` | Yes | — | Redis master name monitored by Sentinel | +| `sentinelAddrs` | `[]string` | One of addrs/service | — | Direct Sentinel host:port addresses | +| `sentinelService` | `SentinelServiceRef` | One of addrs/service | — | Kubernetes Service for Sentinel discovery | +| `db` | `int32` | No | `0` | Redis database number | + +### SentinelServiceRef (CRD) + +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `name` | `string` | Yes | — | Name of the Kubernetes Service | +| `namespace` | `string` | No | Same namespace | Namespace of the Service | +| `port` | `int32` | No | `26379` | Port of the Sentinel service | + +### RedisACLUserConfig (CRD) + +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `usernameSecretRef` | `SecretKeyRef` | Yes | — | Secret reference for Redis username | +| `passwordSecretRef` | `SecretKeyRef` | Yes | — | Secret reference for Redis password | + +## Related Documentation + +- [Architecture Overview](arch/00-overview.md) +- [Operator Architecture](arch/09-operator-architecture.md) +- [Auth Server Storage Architecture](arch/11-auth-server-storage.md) diff --git a/examples/operator/redis-storage/mcpexternalauthconfig-redis-storage.yaml b/examples/operator/redis-storage/mcpexternalauthconfig-redis-storage.yaml new file mode 100644 index 0000000000..65e68198d3 --- /dev/null +++ b/examples/operator/redis-storage/mcpexternalauthconfig-redis-storage.yaml @@ -0,0 +1,48 @@ +# MCPExternalAuthConfig with Redis Sentinel storage for the embedded auth server. +# This example uses Kubernetes Service discovery to find Sentinel instances. +# +# Prerequisites: +# 1. A running Redis Sentinel deployment with a Sentinel Service: +# - Spotahome operator: see redis-failover.yaml (creates "rfs-redis-sentinel" automatically) +# - Manual setup: see sentinel-service.yaml +# 2. Redis ACL user configured (see redis-credentials.yaml) +# 3. An upstream IDP client configured +# +# Usage: +# kubectl apply -f redis-credentials.yaml +# kubectl apply -f mcpexternalauthconfig-redis-storage.yaml +apiVersion: toolhive.stacklok.dev/v1alpha1 +kind: MCPExternalAuthConfig +metadata: + name: auth-with-redis + namespace: default +spec: + type: embeddedAuthServer + embeddedAuthServer: + issuer: "https://auth.example.com" + upstreamProviders: + - name: google + type: oidc + oidcConfig: + issuerUrl: https://accounts.google.com + clientId: "your-google-client-id" + clientSecretRef: + name: google-oauth-secret + key: client-secret + + storage: + type: redis + redis: + sentinelConfig: + masterName: mymaster + # Discover Sentinels via Kubernetes Service + sentinelService: + name: rfs-redis-sentinel + namespace: redis + aclUserConfig: + usernameSecretRef: + name: redis-credentials + key: username + passwordSecretRef: + name: redis-credentials + key: password diff --git a/examples/operator/redis-storage/redis-credentials.yaml b/examples/operator/redis-storage/redis-credentials.yaml new file mode 100644 index 0000000000..7c08123113 --- /dev/null +++ b/examples/operator/redis-storage/redis-credentials.yaml @@ -0,0 +1,18 @@ +# Kubernetes Secret containing Redis ACL user credentials. +# Used by MCPExternalAuthConfig to authenticate to Redis. +# +# IMPORTANT: Replace the password with a strong, randomly generated value. +# In production, use a secrets management tool (e.g., Sealed Secrets, +# External Secrets Operator, or Vault) instead of plaintext manifests. +# +# The corresponding Redis ACL entry should be: +# user toolhive-auth on > ~thv:auth:* &* +@all +apiVersion: v1 +kind: Secret +metadata: + name: redis-credentials + namespace: default +type: Opaque +stringData: + username: toolhive-auth + password: "CHANGE-ME-use-a-strong-random-password" diff --git a/examples/operator/redis-storage/redis-failover.yaml b/examples/operator/redis-storage/redis-failover.yaml new file mode 100644 index 0000000000..e7dd8b55fc --- /dev/null +++ b/examples/operator/redis-storage/redis-failover.yaml @@ -0,0 +1,70 @@ +# Spotahome Redis Operator - RedisFailover resource +# Deploys a Redis master-replica set with Sentinel monitoring. +# +# Prerequisites: +# 1. Install the Spotahome Redis Operator: +# helm repo add redis-operator https://spotahome.github.io/redis-operator +# helm install redis-operator redis-operator/redis-operator \ +# --namespace redis-operator --create-namespace +# 2. Create the target namespace: +# kubectl create namespace redis +# +# Usage: +# kubectl apply -f redis-failover.yaml +# +# This creates: +# - 3 Redis replicas (1 master + 2 replicas) with persistent storage +# - 3 Sentinel instances for automatic failover +# - Service "rfs-redis" (redis namespace) — Redis master, port 6379 +# - Service "rfs-redis-sentinel" (redis namespace) — Sentinel, port 26379 +# +# The "rfs-redis-sentinel" Service is what mcpexternalauthconfig-redis-storage.yaml +# references via sentinelService. The operator resolves its EndpointSlices to +# discover individual Sentinel pod addresses. +# +# If you are NOT using the Spotahome operator, see sentinel-service.yaml for +# how to create this Service manually. +apiVersion: databases.spotahome.com/v1 +kind: RedisFailover +metadata: + name: redis + namespace: redis +spec: + sentinel: + replicas: 3 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + redis: + replicas: 3 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + customConfig: + # Enable ACL file for user management. + # IMPORTANT: You must provision /data/users.acl on each Redis pod + # before authentication will work. See Step 3 ("Configure Redis ACL + # Users") in docs/redis-storage.md for the ACL entry format. + # Common approaches: + # - Init container that writes the ACL file from a Secret/ConfigMap + # - Spotahome operator's extraVolumes/extraVolumeMounts + # - redis-cli ACL SETUSER command via a Job after deployment + - "aclfile /data/users.acl" + storage: + persistentVolumeClaim: + metadata: + name: redis-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/examples/operator/redis-storage/sentinel-service.yaml b/examples/operator/redis-storage/sentinel-service.yaml new file mode 100644 index 0000000000..0b96ebbd29 --- /dev/null +++ b/examples/operator/redis-storage/sentinel-service.yaml @@ -0,0 +1,121 @@ +# Manual Sentinel Service for non-Spotahome deployments. +# +# If you are using the Spotahome Redis Operator (redis-failover.yaml), you do NOT +# need this file — the operator automatically creates a "rfs-redis-sentinel" Service. +# +# This example shows how to create the Sentinel Service manually when running +# Redis Sentinel outside of the Spotahome operator (e.g., a StatefulSet, Helm +# chart, or externally managed Redis). The MCPExternalAuthConfig sentinelService +# field resolves endpoints from this Service via EndpointSlices. +# +# Prerequisites: +# 1. Redis Sentinel pods running with a known label selector +# 2. Create the target namespace: kubectl create namespace redis +# +# Usage: +# kubectl apply -f sentinel-service.yaml + +--- +# Headless Service that selects your Sentinel pods. +# Adjust the selector labels to match your Sentinel deployment. +apiVersion: v1 +kind: Service +metadata: + name: redis-sentinel + namespace: redis + labels: + app: redis-sentinel +spec: + # ClusterIP: None makes this headless — each Sentinel pod gets its own + # EndpointSlice entry, which is how the ToolHive operator discovers them. + clusterIP: None + ports: + - name: sentinel + port: 26379 + targetPort: 26379 + protocol: TCP + selector: + # Change these labels to match your Sentinel pod labels. + app: redis-sentinel + +--- +# Example StatefulSet running 3 Redis Sentinel instances. +# This is one way to deploy Sentinel manually. Adapt as needed for your +# environment (e.g., different image, config, or volume mounts). +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis-sentinel + namespace: redis +spec: + serviceName: redis-sentinel + replicas: 3 + selector: + matchLabels: + app: redis-sentinel + template: + metadata: + labels: + app: redis-sentinel + spec: + # Sentinel modifies its config at runtime, so copy from the + # read-only ConfigMap to a writable location before starting. + initContainers: + - name: copy-config + image: redis:7-alpine + command: ["cp", "/etc/redis-ro/sentinel.conf", "/data/sentinel.conf"] + volumeMounts: + - name: sentinel-config + mountPath: /etc/redis-ro + - name: sentinel-data + mountPath: /data + containers: + - name: sentinel + image: redis:7-alpine + ports: + - containerPort: 26379 + name: sentinel + command: + - redis-sentinel + - /data/sentinel.conf + volumeMounts: + - name: sentinel-data + mountPath: /data + readinessProbe: + exec: + command: + - redis-cli + - -p + - "26379" + - ping + initialDelaySeconds: 5 + periodSeconds: 5 + volumes: + - name: sentinel-config + configMap: + name: redis-sentinel-config + volumeClaimTemplates: + - metadata: + name: sentinel-data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 100Mi + +--- +# Sentinel configuration ConfigMap. +# Update "sentinel monitor" with your Redis master address. +apiVersion: v1 +kind: ConfigMap +metadata: + name: redis-sentinel-config + namespace: redis +data: + sentinel.conf: | + # Monitor the Redis master named "mymaster" at the given address. + # The "2" means quorum: 2 out of 3 Sentinels must agree for failover. + sentinel monitor mymaster redis-master.redis.svc.cluster.local 6379 2 + sentinel down-after-milliseconds mymaster 5000 + sentinel failover-timeout mymaster 10000 + sentinel parallel-syncs mymaster 1 From cedda09b672ffeccca8c651fe240db8e3b79605c Mon Sep 17 00:00:00 2001 From: Trey Date: Mon, 2 Mar 2026 08:31:46 -0800 Subject: [PATCH 2/3] Address feedback --- docs/arch/11-auth-server-storage.md | 2 +- docs/redis-storage.md | 13 +++++++++---- .../operator/redis-storage/sentinel-service.yaml | 5 +++++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/docs/arch/11-auth-server-storage.md b/docs/arch/11-auth-server-storage.md index dc215a5f91..78168729de 100644 --- a/docs/arch/11-auth-server-storage.md +++ b/docs/arch/11-auth-server-storage.md @@ -139,7 +139,7 @@ All values are stored as JSON. The implementation uses defensive copies on read ### TTL Management -Redis TTL (`SETEX`) is used for all time-bounded data. TTL values are derived from OAuth 2.0 token lifetimes: +Redis TTL is used for all time-bounded data. TTL values are derived from OAuth 2.0 token lifetimes: | Data Type | Default TTL | |---|---| diff --git a/docs/redis-storage.md b/docs/redis-storage.md index 35aaea009f..e325808c88 100644 --- a/docs/redis-storage.md +++ b/docs/redis-storage.md @@ -20,6 +20,9 @@ By default, ToolHive's embedded auth server uses in-memory storage. This works w ## Configuration +> **Note: No TLS support.** Redis connections are currently unencrypted. All traffic — including OAuth tokens, authorization codes, and credentials — is transmitted in plaintext between the auth server and Redis. +> In shared network environments, use Kubernetes NetworkPolicies to restrict access to Redis pods, or deploy a service mesh (e.g., Istio, Linkerd) for transparent mTLS. TLS support is planned as a future enhancement. + ### Kubernetes (MCPExternalAuthConfig CRD) When using the ToolHive operator, Redis storage is configured through the `storage` field in the embedded auth server section of `MCPExternalAuthConfig`. @@ -202,17 +205,19 @@ Create a ConfigMap or init container to provision the ACL file. The ACL user nee ``` # /data/users.acl -user toolhive-auth on > ~thv:auth:* &* +@all +user toolhive-auth on > ~thv:auth:* &* +GET +SET +DEL +EXISTS +EXPIRE +SADD +SREM +SMEMBERS +EVAL +MULTI +EXEC +PING ``` This ACL entry: - `on` — Enables the user - `>` — Sets the password - `~thv:auth:*` — Allows access to all keys with the `thv:auth:` prefix -- `&*` — Allows access to all Pub/Sub channels (required for Sentinel) -- `+@all` — Allows all commands +- `&*` — Allows access to all Pub/Sub channels; required by the go-redis Sentinel client to receive `+switch-master` failover notifications. In a multi-tenant Redis deployment, consider restricting this to specific channels if your Redis version supports it. +- `+GET +SET +DEL ...` — Grants only the commands used by the ToolHive auth server + +> **Development / quick-start only:** You can replace the explicit command list with `+@all` to allow all commands, but this is not recommended for production environments. -> **Security note:** In production, restrict the allowed commands to the minimum required set. The auth server uses `GET`, `SET`, `DEL`, `EXISTS`, `EXPIRE`, `SADD`, `SREM`, `SMEMBERS`, `EVAL`, `MULTI`, `EXEC`, and `PING`. +> **Security note:** The auth server uses `GET`, `SET`, `DEL`, `EXISTS`, `EXPIRE`, `SADD`, `SREM`, `SMEMBERS`, `EVAL`, `MULTI`, `EXEC`, and `PING`. Restrict the ACL to this set to follow the principle of least privilege. ### Step 4: Create the ToolHive Auth Config diff --git a/examples/operator/redis-storage/sentinel-service.yaml b/examples/operator/redis-storage/sentinel-service.yaml index 0b96ebbd29..bc4ed8b940 100644 --- a/examples/operator/redis-storage/sentinel-service.yaml +++ b/examples/operator/redis-storage/sentinel-service.yaml @@ -113,6 +113,11 @@ metadata: namespace: redis data: sentinel.conf: | + # Required for Kubernetes: resolve and announce hostnames instead of IPs. + # Without these, after a failover Sentinel advertises raw pod IPs which + # become stale when pods reschedule. + sentinel resolve-hostnames yes + sentinel announce-hostnames yes # Monitor the Redis master named "mymaster" at the given address. # The "2" means quorum: 2 out of 3 Sentinels must agree for failover. sentinel monitor mymaster redis-master.redis.svc.cluster.local 6379 2 From 70f574ca4566a741d38bec5ffe33bc2f1deed5e3 Mon Sep 17 00:00:00 2001 From: Trey Date: Mon, 2 Mar 2026 10:42:07 -0800 Subject: [PATCH 3/3] Update examples based on working demo --- .../mcpexternalauthconfig-redis-storage.yaml | 11 +- .../redis-storage/redis-credentials.yaml | 3 +- .../redis-storage/redis-failover.yaml | 46 ++-- .../redis-storage/sentinel-service.yaml | 233 +++++++++++++----- 4 files changed, 204 insertions(+), 89 deletions(-) diff --git a/examples/operator/redis-storage/mcpexternalauthconfig-redis-storage.yaml b/examples/operator/redis-storage/mcpexternalauthconfig-redis-storage.yaml index 65e68198d3..0905328af7 100644 --- a/examples/operator/redis-storage/mcpexternalauthconfig-redis-storage.yaml +++ b/examples/operator/redis-storage/mcpexternalauthconfig-redis-storage.yaml @@ -3,8 +3,9 @@ # # Prerequisites: # 1. A running Redis Sentinel deployment with a Sentinel Service: -# - Spotahome operator: see redis-failover.yaml (creates "rfs-redis-sentinel" automatically) -# - Manual setup: see sentinel-service.yaml +# - Recommended: see sentinel-service.yaml (complete Redis + Sentinel setup) +# - Note: the Spotahome operator (redis-failover.yaml) has known issues; +# see that file for details. # 2. Redis ACL user configured (see redis-credentials.yaml) # 3. An upstream IDP client configured # @@ -35,9 +36,11 @@ spec: redis: sentinelConfig: masterName: mymaster - # Discover Sentinels via Kubernetes Service + # Discover Sentinels via the headless Service created by sentinel-service.yaml. + # The ToolHive operator resolves this Service's EndpointSlices to find + # individual Sentinel pod addresses. sentinelService: - name: rfs-redis-sentinel + name: redis-sentinel namespace: redis aclUserConfig: usernameSecretRef: diff --git a/examples/operator/redis-storage/redis-credentials.yaml b/examples/operator/redis-storage/redis-credentials.yaml index 7c08123113..f94f402035 100644 --- a/examples/operator/redis-storage/redis-credentials.yaml +++ b/examples/operator/redis-storage/redis-credentials.yaml @@ -6,7 +6,8 @@ # External Secrets Operator, or Vault) instead of plaintext manifests. # # The corresponding Redis ACL entry should be: -# user toolhive-auth on > ~thv:auth:* &* +@all +# user toolhive-auth on > ~thv:auth:* &* +GET +SET +SETNX +DEL +EXISTS +EXPIRE +SADD +SREM +SMEMBERS +EVAL +MULTI +EXEC +EVALSHA +PING +# (see sentinel-service.yaml for the full ACL Secret that provisions this into Redis) apiVersion: v1 kind: Secret metadata: diff --git a/examples/operator/redis-storage/redis-failover.yaml b/examples/operator/redis-storage/redis-failover.yaml index e7dd8b55fc..449eb522f8 100644 --- a/examples/operator/redis-storage/redis-failover.yaml +++ b/examples/operator/redis-storage/redis-failover.yaml @@ -1,29 +1,35 @@ # Spotahome Redis Operator - RedisFailover resource -# Deploys a Redis master-replica set with Sentinel monitoring. # -# Prerequisites: -# 1. Install the Spotahome Redis Operator: -# helm repo add redis-operator https://spotahome.github.io/redis-operator -# helm install redis-operator redis-operator/redis-operator \ -# --namespace redis-operator --create-namespace -# 2. Create the target namespace: -# kubectl create namespace redis +# WARNING: The Spotahome Redis Operator has known issues that make it +# unsuitable for this use case. Use sentinel-service.yaml instead. +# +# Known issues: # -# Usage: -# kubectl apply -f redis-failover.yaml +# 1. Helm chart 3.3.0+ fails to install its own CRD: +# "failed to install CRD: error converting YAML to JSON: did not find +# expected node content" +# Workaround: pin to chart 3.2.9 or apply the CRD manually. +# See: https://github.com/spotahome/redis-operator/issues/679 # -# This creates: -# - 3 Redis replicas (1 master + 2 replicas) with persistent storage -# - 3 Sentinel instances for automatic failover -# - Service "rfs-redis" (redis namespace) — Redis master, port 6379 -# - Service "rfs-redis-sentinel" (redis namespace) — Sentinel, port 26379 +# 2. Sentinel advertises 127.0.0.1 as the Redis master address. +# The operator configures Sentinel to initially monitor 127.0.0.1:6379. +# Because sentinel.conf is generated internally by the operator, adding +# "sentinel resolve-hostnames yes" / "sentinel announce-hostnames yes" +# via customConfig does not reliably fix this. Clients in other pods +# receive 127.0.0.1 as the master address and cannot connect. # -# The "rfs-redis-sentinel" Service is what mcpexternalauthconfig-redis-storage.yaml -# references via sentinelService. The operator resolves its EndpointSlices to -# discover individual Sentinel pod addresses. +# This file is retained for reference only. For a working Redis Sentinel +# deployment, see sentinel-service.yaml. # -# If you are NOT using the Spotahome operator, see sentinel-service.yaml for -# how to create this Service manually. +# ───────────────────────────────────────────────────────────────────────────── +# +# Original prerequisites (if you still want to try this approach): +# 1. Install the Spotahome Redis Operator (pin to 3.2.9): +# helm repo add redis-operator https://spotahome.github.io/redis-operator +# helm install redis-operator redis-operator/redis-operator \ +# --version 3.2.9 --namespace redis-operator --create-namespace +# 2. Create the target namespace: +# kubectl create namespace redis apiVersion: databases.spotahome.com/v1 kind: RedisFailover metadata: diff --git a/examples/operator/redis-storage/sentinel-service.yaml b/examples/operator/redis-storage/sentinel-service.yaml index bc4ed8b940..d6e4fb7c0d 100644 --- a/examples/operator/redis-storage/sentinel-service.yaml +++ b/examples/operator/redis-storage/sentinel-service.yaml @@ -1,47 +1,172 @@ -# Manual Sentinel Service for non-Spotahome deployments. +# Complete Redis + Sentinel deployment for ToolHive auth server token storage. # -# If you are using the Spotahome Redis Operator (redis-failover.yaml), you do NOT -# need this file — the operator automatically creates a "rfs-redis-sentinel" Service. +# This is the recommended approach. The Spotahome Redis Operator (redis-failover.yaml) +# has known issues that make it unsuitable for this use case — see redis-failover.yaml +# for details. # -# This example shows how to create the Sentinel Service manually when running -# Redis Sentinel outside of the Spotahome operator (e.g., a StatefulSet, Helm -# chart, or externally managed Redis). The MCPExternalAuthConfig sentinelService -# field resolves endpoints from this Service via EndpointSlices. +# What this creates (all in the "redis" namespace): +# - redis-acl Secret — ACL file provisioned into each Redis pod +# - redis Service — headless; gives Redis pods stable DNS names +# - redis StatefulSet — 1 Redis pod (redis-0.redis.redis.svc.cluster.local) +# - redis-sentinel-config ConfigMap — sentinel.conf with hostname resolution +# - redis-sentinel Service — headless; required for Sentinel announce-hostnames +# - redis-sentinel StatefulSet — 3 Sentinel pods +# +# The "redis-sentinel" headless Service is referenced by sentinelService in +# mcpexternalauthconfig-redis-storage.yaml. The ToolHive operator resolves its +# EndpointSlices to discover individual Sentinel pod addresses. # # Prerequisites: -# 1. Redis Sentinel pods running with a known label selector -# 2. Create the target namespace: kubectl create namespace redis +# kubectl create namespace redis # # Usage: +# # Fill in your Redis password, then apply: +# REDIS_PASSWORD= envsubst < sentinel-service.yaml | kubectl apply -f - +# +# # Or substitute manually and apply directly: # kubectl apply -f sentinel-service.yaml --- -# Headless Service that selects your Sentinel pods. -# Adjust the selector labels to match your Sentinel deployment. +# ACL file provisioned into each Redis pod by the init container. +# Fill in the password before applying (must match redis-credentials.yaml). +# +# The ACL entry grants the toolhive-auth user access to: +# ~thv:auth:* — keys with the ToolHive auth prefix +# &* — all Pub/Sub channels (required for Sentinel failover notifications) +# +GET +SET … — only the commands the auth server uses (principle of least privilege) +apiVersion: v1 +kind: Secret +metadata: + name: redis-acl + namespace: redis +type: Opaque +stringData: + users.acl: "user toolhive-auth on > ~thv:auth:* &* +GET +SET +SETNX +DEL +EXISTS +EXPIRE +SADD +SREM +SMEMBERS +EVAL +MULTI +EXEC +EVALSHA +PING" +--- +# Headless Service gives Redis pods stable, individually addressable DNS names: +# redis-0.redis.redis.svc.cluster.local +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: redis +spec: + clusterIP: None + selector: + app: redis + ports: + - name: redis + port: 6379 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis + namespace: redis +spec: + serviceName: redis + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + initContainers: + # Copy the ACL Secret (read-only mount) to the writable data volume so + # Redis can load and rewrite it via the "aclfile" directive. + - name: init-acl + image: redis:7-alpine + command: ["cp", "/etc/redis-acl/users.acl", "/data/users.acl"] + volumeMounts: + - name: redis-acl + mountPath: /etc/redis-acl + - name: redis-data + mountPath: /data + containers: + - name: redis + image: redis:7-alpine + ports: + - containerPort: 6379 + command: + - redis-server + - --bind + - "0.0.0.0" + - --aclfile + - /data/users.acl + readinessProbe: + exec: + command: ["redis-cli", "PING"] + initialDelaySeconds: 5 + periodSeconds: 5 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + volumeMounts: + - name: redis-data + mountPath: /data + - name: redis-acl + mountPath: /etc/redis-acl + readOnly: true + volumes: + - name: redis-acl + secret: + secretName: redis-acl + volumeClaimTemplates: + - metadata: + name: redis-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +# sentinel.conf for all Sentinel pods. +# +# resolve-hostnames and announce-hostnames are required in Kubernetes. +# Without them, Sentinel advertises 127.0.0.1 as the master address, which is +# unreachable from other pods. +apiVersion: v1 +kind: ConfigMap +metadata: + name: redis-sentinel-config + namespace: redis +data: + sentinel.conf: | + sentinel resolve-hostnames yes + sentinel announce-hostnames yes + # Monitor the Redis master by its stable StatefulSet DNS name. + # The "2" means quorum: 2 out of 3 Sentinels must agree for failover. + sentinel monitor mymaster redis-0.redis.redis.svc.cluster.local 6379 2 + sentinel down-after-milliseconds mymaster 5000 + sentinel failover-timeout mymaster 10000 + sentinel parallel-syncs mymaster 1 +--- +# Headless Service for Sentinel pods. Required for two reasons: +# 1. Gives pods stable DNS names used by "sentinel announce-hostnames yes" +# (e.g., redis-sentinel-0.redis-sentinel.redis.svc.cluster.local) +# 2. Referenced by sentinelService in MCPExternalAuthConfig — the ToolHive +# operator uses this Service's EndpointSlices to discover Sentinel pods. apiVersion: v1 kind: Service metadata: name: redis-sentinel namespace: redis - labels: - app: redis-sentinel spec: - # ClusterIP: None makes this headless — each Sentinel pod gets its own - # EndpointSlice entry, which is how the ToolHive operator discovers them. clusterIP: None + selector: + app: redis-sentinel ports: - name: sentinel port: 26379 - targetPort: 26379 - protocol: TCP - selector: - # Change these labels to match your Sentinel pod labels. - app: redis-sentinel - --- -# Example StatefulSet running 3 Redis Sentinel instances. -# This is one way to deploy Sentinel manually. Adapt as needed for your -# environment (e.g., different image, config, or volume mounts). apiVersion: apps/v1 kind: StatefulSet metadata: @@ -58,15 +183,15 @@ spec: labels: app: redis-sentinel spec: - # Sentinel modifies its config at runtime, so copy from the - # read-only ConfigMap to a writable location before starting. initContainers: + # Sentinel rewrites sentinel.conf at runtime, so copy from the read-only + # ConfigMap to a writable PVC-backed volume before starting. - name: copy-config image: redis:7-alpine - command: ["cp", "/etc/redis-ro/sentinel.conf", "/data/sentinel.conf"] + command: ["cp", "/etc/sentinel-ro/sentinel.conf", "/data/sentinel.conf"] volumeMounts: - - name: sentinel-config - mountPath: /etc/redis-ro + - name: sentinel-config-ro + mountPath: /etc/sentinel-ro - name: sentinel-data mountPath: /data containers: @@ -75,52 +200,32 @@ spec: ports: - containerPort: 26379 name: sentinel - command: - - redis-sentinel - - /data/sentinel.conf - volumeMounts: - - name: sentinel-data - mountPath: /data + command: ["redis-sentinel", "/data/sentinel.conf"] readinessProbe: exec: - command: - - redis-cli - - -p - - "26379" - - ping + command: ["redis-cli", "-p", "26379", "PING"] initialDelaySeconds: 5 periodSeconds: 5 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + volumeMounts: + - name: sentinel-data + mountPath: /data volumes: - - name: sentinel-config + - name: sentinel-config-ro configMap: name: redis-sentinel-config volumeClaimTemplates: - metadata: name: sentinel-data spec: - accessModes: ["ReadWriteOnce"] + accessModes: + - ReadWriteOnce resources: requests: storage: 100Mi - ---- -# Sentinel configuration ConfigMap. -# Update "sentinel monitor" with your Redis master address. -apiVersion: v1 -kind: ConfigMap -metadata: - name: redis-sentinel-config - namespace: redis -data: - sentinel.conf: | - # Required for Kubernetes: resolve and announce hostnames instead of IPs. - # Without these, after a failover Sentinel advertises raw pod IPs which - # become stale when pods reschedule. - sentinel resolve-hostnames yes - sentinel announce-hostnames yes - # Monitor the Redis master named "mymaster" at the given address. - # The "2" means quorum: 2 out of 3 Sentinels must agree for failover. - sentinel monitor mymaster redis-master.redis.svc.cluster.local 6379 2 - sentinel down-after-milliseconds mymaster 5000 - sentinel failover-timeout mymaster 10000 - sentinel parallel-syncs mymaster 1