uber · sbalabanov · Jul 23, 2026 · Jul 23, 2026
@@ -19,18 +19,23 @@ services:
     environment:
       MYSQL_ROOT_PASSWORD: root
       MYSQL_DATABASE: submitqueue
+      MYSQL_INITDB_SKIP_TZINFO: ${SQ_MYSQL_INITDB_SKIP_TZINFO:-}
     ports:
       - "3306" # Random ephemeral port to avoid conflicts
+    volumes:
+      - type: ${SQ_MYSQL_DATA_MOUNT_TYPE:-volume}
+        target: /var/lib/mysql
     healthcheck:
       # Use 127.0.0.1 (TCP) instead of localhost (Unix socket). MySQL treats
       # "localhost" as a socket connection, which can be ready before the TCP
       # listener — causing dependent services that connect over TCP to fail.
       test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-proot"]
-      interval: 5s
-      timeout: 5s
-      retries: 10
+      interval: 1s
+      timeout: 2s
+      retries: 60
 
   runway-service:
+    image: ${SQ_DOCKER_IMAGE_PREFIX:-submitqueue}-runway-service:latest
     build:
       context: ${REPO_ROOT}
       dockerfile: service/runway/server/Dockerfile

@@ -23,32 +23,41 @@ services:
     environment:
       MYSQL_ROOT_PASSWORD: root
       MYSQL_DATABASE: submitqueue
+      MYSQL_INITDB_SKIP_TZINFO: ${SQ_MYSQL_INITDB_SKIP_TZINFO:-}
     ports:
       - "3306" # Random ephemeral port to avoid conflicts
+    volumes:
+      - type: ${SQ_MYSQL_DATA_MOUNT_TYPE:-volume}
+        target: /var/lib/mysql
     healthcheck:
       # Use 127.0.0.1 (TCP) instead of localhost (Unix socket). MySQL treats
       # "localhost" as a socket connection, which can be ready before the TCP
       # listener — causing dependent services that connect over TCP to fail.
       test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-proot"]
-      interval: 5s
-      timeout: 5s
-      retries: 10
+      interval: 1s
+      timeout: 2s
+      retries: 60
 
   # Queue database - messaging infrastructure (messages, offsets, partition leases).
   mysql-queue:
     image: mysql:8.0
     environment:
       MYSQL_ROOT_PASSWORD: root
       MYSQL_DATABASE: submitqueue
+      MYSQL_INITDB_SKIP_TZINFO: ${SQ_MYSQL_INITDB_SKIP_TZINFO:-}
     ports:
       - "3306" # Random ephemeral port to avoid conflicts
+    volumes:
+      - type: ${SQ_MYSQL_DATA_MOUNT_TYPE:-volume}
+        target: /var/lib/mysql
     healthcheck:
       test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-proot"]
-      interval: 5s
-      timeout: 5s
-      retries: 10
+      interval: 1s
+      timeout: 2s
+      retries: 60
 
   stovepipe-service:
+    image: ${SQ_DOCKER_IMAGE_PREFIX:-submitqueue}-stovepipe-service:latest
     build:
       context: ${REPO_ROOT}
       dockerfile: service/stovepipe/server/Dockerfile

@@ -15,16 +15,20 @@ services:
     environment:
       MYSQL_ROOT_PASSWORD: root
       MYSQL_DATABASE: submitqueue
+      MYSQL_INITDB_SKIP_TZINFO: ${SQ_MYSQL_INITDB_SKIP_TZINFO:-}
     ports:
       - "3306" # Random ephemeral port to avoid conflicts
+    volumes:
+      - type: ${SQ_MYSQL_DATA_MOUNT_TYPE:-volume}
+        target: /var/lib/mysql
     healthcheck:
       # Use 127.0.0.1 (TCP) instead of localhost (Unix socket). MySQL treats
       # "localhost" as a socket connection, which can be ready before the TCP
       # listener — causing dependent services that connect over TCP to fail.
       test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-proot"]
-      interval: 5s
-      timeout: 5s
-      retries: 10
+      interval: 1s
+      timeout: 2s
+      retries: 60
 
   # Queue Database - Messaging infrastructure (messages, offsets, partition leases)
   # Separate from app DB to demonstrate queue is pluggable infrastructure
@@ -33,15 +37,20 @@ services:
     environment:
       MYSQL_ROOT_PASSWORD: root
       MYSQL_DATABASE: submitqueue
+      MYSQL_INITDB_SKIP_TZINFO: ${SQ_MYSQL_INITDB_SKIP_TZINFO:-}
     ports:
       - "3306" # Random ephemeral port to avoid conflicts
+    volumes:
+      - type: ${SQ_MYSQL_DATA_MOUNT_TYPE:-volume}
+        target: /var/lib/mysql
     healthcheck:
       test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-proot"]
-      interval: 5s
-      timeout: 5s
-      retries: 10
+      interval: 1s
+      timeout: 2s
+      retries: 60
 
   gateway-service:
+    image: ${SQ_DOCKER_IMAGE_PREFIX:-submitqueue}-gateway-service:latest
     build:
       context: ${REPO_ROOT}
       dockerfile: service/submitqueue/gateway/server/Dockerfile
@@ -64,6 +73,7 @@ services:
         condition: service_healthy
 
   orchestrator-service:
+    image: ${SQ_DOCKER_IMAGE_PREFIX:-submitqueue}-orchestrator-service:latest
     build:
       context: ${REPO_ROOT}
       dockerfile: service/submitqueue/orchestrator/server/Dockerfile
@@ -88,6 +98,7 @@ services:
   # needs the queue DB (shared with the orchestrator over mysql-queue); it has
   # no application database of its own.
   runway-service:
+    image: ${SQ_DOCKER_IMAGE_PREFIX:-submitqueue}-runway-service:latest
     build:
       context: ${REPO_ROOT}
       dockerfile: service/runway/server/Dockerfile

@@ -15,16 +15,20 @@ services:
     environment:
       MYSQL_ROOT_PASSWORD: root
       MYSQL_DATABASE: submitqueue
+      MYSQL_INITDB_SKIP_TZINFO: ${SQ_MYSQL_INITDB_SKIP_TZINFO:-}
     ports:
       - "3306" # Random ephemeral port to avoid conflicts
+    volumes:
+      - type: ${SQ_MYSQL_DATA_MOUNT_TYPE:-volume}
+        target: /var/lib/mysql
     healthcheck:
       # Use 127.0.0.1 (TCP) instead of localhost (Unix socket). MySQL treats
       # "localhost" as a socket connection, which can be ready before the TCP
       # listener — causing dependent services that connect over TCP to fail.
       test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-proot"]
-      interval: 5s
-      timeout: 5s
-      retries: 10
+      interval: 1s
+      timeout: 2s
+      retries: 60
 
   # Queue Database - Messaging infrastructure (messages, offsets, partition leases)
   # Separate from app DB to demonstrate queue is pluggable infrastructure
@@ -33,15 +37,20 @@ services:
     environment:
       MYSQL_ROOT_PASSWORD: root
       MYSQL_DATABASE: submitqueue
+      MYSQL_INITDB_SKIP_TZINFO: ${SQ_MYSQL_INITDB_SKIP_TZINFO:-}
     ports:
       - "3306" # Random ephemeral port to avoid conflicts
+    volumes:
+      - type: ${SQ_MYSQL_DATA_MOUNT_TYPE:-volume}
+        target: /var/lib/mysql
     healthcheck:
       test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-proot"]
-      interval: 5s
-      timeout: 5s
-      retries: 10
+      interval: 1s
+      timeout: 2s
+      retries: 60
 
   gateway-service:
+    image: ${SQ_DOCKER_IMAGE_PREFIX:-submitqueue}-gateway-service:latest
     build:
       context: ${REPO_ROOT}
       dockerfile: service/submitqueue/gateway/server/Dockerfile

@@ -15,16 +15,20 @@ services:
     environment:
       MYSQL_ROOT_PASSWORD: root
       MYSQL_DATABASE: submitqueue
+      MYSQL_INITDB_SKIP_TZINFO: ${SQ_MYSQL_INITDB_SKIP_TZINFO:-}
     ports:
       - "3306" # Random ephemeral port to avoid conflicts
+    volumes:
+      - type: ${SQ_MYSQL_DATA_MOUNT_TYPE:-volume}
+        target: /var/lib/mysql
     healthcheck:
       # Use 127.0.0.1 (TCP) instead of localhost (Unix socket). MySQL treats
       # "localhost" as a socket connection, which can be ready before the TCP
       # listener — causing dependent services that connect over TCP to fail.
       test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-proot"]
-      interval: 5s
-      timeout: 5s
-      retries: 10
+      interval: 1s
+      timeout: 2s
+      retries: 60
 
   # Queue Database - Messaging infrastructure (messages, offsets, partition leases)
   # Separate from app DB to demonstrate queue is pluggable infrastructure
@@ -33,15 +37,20 @@ services:
     environment:
       MYSQL_ROOT_PASSWORD: root
       MYSQL_DATABASE: submitqueue
+      MYSQL_INITDB_SKIP_TZINFO: ${SQ_MYSQL_INITDB_SKIP_TZINFO:-}
     ports:
       - "3306" # Random ephemeral port to avoid conflicts
+    volumes:
+      - type: ${SQ_MYSQL_DATA_MOUNT_TYPE:-volume}
+        target: /var/lib/mysql
     healthcheck:
       test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-proot"]
-      interval: 5s
-      timeout: 5s
-      retries: 10
+      interval: 1s
+      timeout: 2s
+      retries: 60
 
   orchestrator-service:
+    image: ${SQ_DOCKER_IMAGE_PREFIX:-submitqueue}-orchestrator-service:latest
     build:
       context: ${REPO_ROOT}
       dockerfile: service/submitqueue/orchestrator/server/Dockerfile

@@ -5,6 +5,8 @@ go_test(
     srcs = ["counter_test.go"],
     data = [
         "docker-compose.yml",
+        "//:MODULE.bazel",
+        "//:go.mod",
         "//platform/extension/counter/mysql/schema",
     ],
     tags = [

@@ -8,13 +8,17 @@ services:
     environment:
       MYSQL_ROOT_PASSWORD: root
       MYSQL_DATABASE: submitqueue
+      MYSQL_INITDB_SKIP_TZINFO: ${SQ_MYSQL_INITDB_SKIP_TZINFO:-}
     ports:
       - "3306" # Random ephemeral port to avoid conflicts
+    volumes:
+      - type: ${SQ_MYSQL_DATA_MOUNT_TYPE:-volume}
+        target: /var/lib/mysql
     healthcheck:
       # Use 127.0.0.1 (TCP) instead of localhost (Unix socket). MySQL treats
       # "localhost" as a socket connection, which can be ready before the TCP
       # listener — causing dependent services that connect over TCP to fail.
       test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-proot"]
-      interval: 5s
-      timeout: 5s
-      retries: 10
+      interval: 1s
+      timeout: 2s
+      retries: 60
@@ -5,6 +5,8 @@ go_test(
     srcs = ["queue_test.go"],
     data = [
         "docker-compose.yml",
+        "//:MODULE.bazel",
+        "//:go.mod",
         "//platform/extension/messagequeue/mysql/schema",
     ],
     tags = [

@@ -8,13 +8,17 @@ services:
     environment:
       MYSQL_ROOT_PASSWORD: root
       MYSQL_DATABASE: submitqueue
+      MYSQL_INITDB_SKIP_TZINFO: ${SQ_MYSQL_INITDB_SKIP_TZINFO:-}
     ports:
       - "3306" # Random ephemeral port to avoid conflicts
+    volumes:
+      - type: ${SQ_MYSQL_DATA_MOUNT_TYPE:-volume}
+        target: /var/lib/mysql
     healthcheck:
       # Use 127.0.0.1 (TCP) instead of localhost (Unix socket). MySQL treats
       # "localhost" as a socket connection, which can be ready before the TCP
       # listener — causing dependent services that connect over TCP to fail.
       test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-proot"]
-      interval: 5s
-      timeout: 5s
-      retries: 10
+      interval: 1s
+      timeout: 2s
+      retries: 60
@@ -291,6 +291,40 @@ func waitForCondition(t *testing.T, signalCh <-chan queueMySQL.HookSignal, condi
 	}
 }
 
+// waitForLag waits for subscriber polls until the selected consumer lag
+// converges to expected. Lag must not advance past the expected value.
+func waitForLag(
+	t *testing.T,
+	ctx context.Context,
+	admin *queueAdmin.AdminStore,
+	signalCh <-chan queueMySQL.HookSignal,
+	topic string,
+	consumerGroup string,
+	partitionKey string,
+	expected int64,
+) int64 {
+	t.Helper()
+
+	for {
+		lags, err := admin.ConsumerLag(ctx, topic)
+		require.NoError(t, err)
+
+		var actual int64 = -1
+		for _, lag := range lags {
+			if lag.ConsumerGroup == consumerGroup && lag.PartitionKey == partitionKey {
+				actual = lag.Lag
+				break
+			}
+		}
+
+		require.GreaterOrEqual(t, actual, expected, "watermark advanced past expected lag")
+		if actual == expected {
+			return actual
+		}
+		waitForSignal(t, signalCh, queueMySQL.SignalDeliveryCheck)
+	}
+}
+
 func (s *SQLQueueIntegrationSuite) TestPublishAndSubscribe() {
 	t := s.T()
 
@@ -2396,18 +2430,9 @@ func (s *SQLQueueIntegrationSuite) TestWatermarkAdvancesContiguously() {
 
 	admin := queueAdmin.NewAdminStore(s.db)
 
-	// Helper to get consumer lag
-	getLag := func() int64 {
-		lags, err := admin.ConsumerLag(s.ctx, topic)
-		require.NoError(t, err)
-		for _, lag := range lags {
-			if lag.ConsumerGroup == "watermark-cg" && lag.PartitionKey == "wm-part" {
-				return lag.Lag
-			}
-		}
-		return -1
-	}
-
+	// Watermark advancement is incremental and runs in the subscriber poll loop.
+	// A delivery-check signal proves one poll completed, but the watermark may
+	// need another poll to converge after several acknowledgements.
 	// Ack message 3 first (out of order)
 	require.NoError(t, deliveries["wm-msg-3"].Ack(s.ctx))
 	t.Logf("Acked msg-3")
@@ -2417,31 +2442,24 @@ func (s *SQLQueueIntegrationSuite) TestWatermarkAdvancesContiguously() {
 	require.NoError(t, deliveries["wm-msg-2"].Ack(s.ctx))
 	t.Logf("Acked msg-1 and msg-2")
 
-	// Wait for poll loop to advance watermark
-	waitForSignal(t, signalCh, queueMySQL.SignalDeliveryCheck)
-
 	// After acking 1,2,3: watermark should advance to 3, lag should be 2 (msg-4, msg-5)
-	lag := getLag()
+	lag := waitForLag(t, s.ctx, admin, signalCh, topic, "watermark-cg", "wm-part", 2)
 	assert.Equal(t, int64(2), lag, "lag should be 2 after acking 1,2,3 (4 and 5 remain)")
 	t.Logf("After acking 1,2,3: lag=%d", lag)
 
 	// Ack message 5 (skip 4) — watermark should NOT advance past 3
 	require.NoError(t, deliveries["wm-msg-5"].Ack(s.ctx))
 	t.Logf("Acked msg-5 (skipping msg-4)")
 
-	waitForSignal(t, signalCh, queueMySQL.SignalDeliveryCheck)
-
-	lag = getLag()
+	lag = waitForLag(t, s.ctx, admin, signalCh, topic, "watermark-cg", "wm-part", 2)
 	assert.Equal(t, int64(2), lag, "lag should still be 2 after acking 5 but not 4")
 	t.Logf("After acking 5 (not 4): lag=%d", lag)
 
 	// Ack message 4 — now all 5 are contiguous, watermark should advance to 5
 	require.NoError(t, deliveries["wm-msg-4"].Ack(s.ctx))
 	t.Logf("Acked msg-4")
 
-	waitForSignal(t, signalCh, queueMySQL.SignalDeliveryCheck)
-
-	lag = getLag()
+	lag = waitForLag(t, s.ctx, admin, signalCh, topic, "watermark-cg", "wm-part", 0)
 	assert.Equal(t, int64(0), lag, "lag should be 0 after acking all 5 messages")
 	t.Logf("After acking all 5: lag=%d", lag)