Skip to content

feat: Adding client-proxy and clickhouse to e2b-artifacts #3210

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
charlie-e2b merged 3 commits into from
Jul 7, 2026
Merged

feat: Adding client-proxy and clickhouse to e2b-artifacts #3210

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8281d8d
feat: Adding client-proxy and clickhouse to e2b-artifacts
charlie-e2b Jul 6, 2026
9501b5f
fix: updating templates
charlie-e2b Jul 6, 2026
d3a5d0a
fix: for claude comments about forcing everything to be amd64
charlie-e2b Jul 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .env.aws.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,13 @@ TERRAFORM_ENVIRONMENT=dev
# ------------------------------------------- Optional block -----------------------------------------------------------
# Following variables are optional and doesn't have to be set

# Pull a released, prebuilt docker-reverse-proxy image from E2B's artifact
# Pull a released, prebuilt docker images from E2B's artifact
# registry instead of building it from source. When set, `make build-and-upload`
# pulls this version and mirrors it into your own core repo (caching it locally).
# Leave empty to build from source. Example: v0.1.0
DOCKER_REVERSE_PROXY_VERSION=
CLIENT_PROXY_VERSION=
CLICKHOUSE_MIGRATOR_VERSION=

# Sandbox firewall: comma-separated CIDRs to allow through the private-range deny list
# ALLOW_SANDBOX_INTERNAL_CIDRS=
4 changes: 3 additions & 1 deletion .env.gcp.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,11 +74,13 @@ CLICKHOUSE_CLUSTER_SIZE=1
# ------------------------------------------- Optional block -----------------------------------------------------------
# Following variables are optional and doesn't have to be set

# Pull a released, prebuilt docker-reverse-proxy image from E2B's artifact
# Pull a released, prebuilt docker images from E2B's artifact
# registry instead of building it from source. When set, `make build-and-upload`
# pulls this version and mirrors it into your own core repo (caching it locally).
# Leave empty to build from source. Example: v0.1.0
DOCKER_REVERSE_PROXY_VERSION=
CLIENT_PROXY_VERSION=
CLICKHOUSE_MIGRATOR_VERSION=

# Dashboard API instance count (default: 0)
DASHBOARD_API_COUNT=
Expand Down
110 changes: 110 additions & 0 deletions .github/workflows/release-please.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,12 @@ jobs:
reverse_proxy_released: ${{ steps.release.outputs['packages/docker-reverse-proxy--release_created'] }}
reverse_proxy_tag: ${{ steps.release.outputs['packages/docker-reverse-proxy--tag_name'] }}
reverse_proxy_version: ${{ steps.release.outputs['packages/docker-reverse-proxy--version'] }}
client_proxy_released: ${{ steps.release.outputs['packages/client-proxy--release_created'] }}
client_proxy_tag: ${{ steps.release.outputs['packages/client-proxy--tag_name'] }}
client_proxy_version: ${{ steps.release.outputs['packages/client-proxy--version'] }}
clickhouse_migrator_released: ${{ steps.release.outputs['packages/clickhouse--release_created'] }}
clickhouse_migrator_tag: ${{ steps.release.outputs['packages/clickhouse--tag_name'] }}
clickhouse_migrator_version: ${{ steps.release.outputs['packages/clickhouse--version'] }}
steps:
# Mint a short-lived token from a GitHub App so the release PR is opened as
# the App (not the default GITHUB_TOKEN). This is required when `main` has
Expand Down Expand Up @@ -111,3 +117,107 @@ jobs:
--push \
-f packages/docker-reverse-proxy/Dockerfile \
packages

publish-client-proxy:
name: Publish client-proxy to e2b-artifacts
needs: release-please
if: ${{ needs.release-please.outputs.client_proxy_released == 'true' }}
runs-on: ubuntu-24.04
permissions:
contents: read
id-token: write
env:
# <host>/<project>/<repository>/<image>
IMAGE: us-docker.pkg.dev/e2b-artifacts/client-proxy/client-proxy
VERSION: ${{ needs.release-please.outputs.client_proxy_version }}
steps:
- name: Checkout repository
uses: actions/checkout@v5
Comment thread
charlie-e2b marked this conversation as resolved.
with:
ref: ${{ github.sha }}
persist-credentials: false

- name: Authenticate to the e2b-artifacts project
uses: google-github-actions/auth@v3
with:
workload_identity_provider: ${{ vars.E2B_ARTIFACTS_WIF_PROVIDER }}
service_account: ${{ vars.E2B_ARTIFACTS_PUBLISH_SA }}

- name: Set up Cloud SDK
uses: google-github-actions/setup-gcloud@v3

- name: Configure Docker auth
run: gcloud auth configure-docker us-docker.pkg.dev --quiet

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Build and push released image
# The Dockerfile lives in packages/client-proxy but its build context is
# the packages/ directory (it needs shared/ and clickhouse/). Same
# tagging conventions as docker-reverse-proxy: v-prefixed tag, no
# :latest (immutableTags=true on the e2b-artifacts repo), and
# --provenance=false --sbom=false for Artifact Registry compatibility.
run: |
IMAGE_TAG="v${VERSION}"
docker buildx build \
--platform linux/amd64 \
--provenance=false \
--sbom=false \
--build-arg COMMIT_SHA="${GITHUB_SHA::7}" \
--tag "${IMAGE}:${IMAGE_TAG}" \
--push \
-f packages/client-proxy/Dockerfile \
packages

publish-clickhouse-migrator:
name: Publish clickhouse-migrator to e2b-artifacts
needs: release-please
if: ${{ needs.release-please.outputs.clickhouse_migrator_released == 'true' }}
runs-on: ubuntu-24.04
permissions:
contents: read
id-token: write
env:
# <host>/<project>/<repository>/<image>
IMAGE: us-docker.pkg.dev/e2b-artifacts/clickhouse-migrator/clickhouse-migrator
VERSION: ${{ needs.release-please.outputs.clickhouse_migrator_version }}
steps:
- name: Checkout repository
uses: actions/checkout@v5
with:
ref: ${{ github.sha }}
persist-credentials: false

- name: Authenticate to the e2b-artifacts project
uses: google-github-actions/auth@v3
with:
workload_identity_provider: ${{ vars.E2B_ARTIFACTS_WIF_PROVIDER }}
service_account: ${{ vars.E2B_ARTIFACTS_PUBLISH_SA }}

- name: Set up Cloud SDK
uses: google-github-actions/setup-gcloud@v3

- name: Configure Docker auth
run: gcloud auth configure-docker us-docker.pkg.dev --quiet

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Build and push released image
# The clickhouse-migrator image is self-contained: its Dockerfile and
# build context both live in packages/clickhouse (matching the package
# Makefile's `docker build .`). Same tagging conventions as
# docker-reverse-proxy: v-prefixed tag, no :latest (immutableTags=true
# on the e2b-artifacts repo), and --provenance=false --sbom=false for
# Artifact Registry compatibility.
run: |
IMAGE_TAG="v${VERSION}"
docker buildx build \
--platform linux/amd64 \
--provenance=false \
--sbom=false \
--tag "${IMAGE}:${IMAGE_TAG}" \
--push \
-f packages/clickhouse/Dockerfile \
packages/clickhouse
4 changes: 3 additions & 1 deletion .release-please-manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
{
"packages/docker-reverse-proxy": "0.2.2"
"packages/docker-reverse-proxy": "0.2.2",
"packages/client-proxy": "0.0.0",

@claude claude Bot Jul 6, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 The client-proxy's telemetry version won't match the release tag. packages/client-proxy/main.go:45 hardcodes const version = "1.2.0", but the release-please manifest starts at 0.0.0 — the first release will produce tags like v0.0.1/v0.1.0 while every published image reports telemetry version 1.2.0. Either seed the manifest at 1.2.0, or wire VERSION through Dockerfile/Makefile/workflow like docker-reverse-proxy does (change const→var, add -X=main.version= ldflag, add ARG VERSION to Dockerfile, pass --build-arg VERSION="${IMAGE_TAG}" in the publish job).

Extended reasoning...

The mismatch

packages/client-proxy/main.go:45 declares:

const (
    serviceName = "client-proxy"
    // ...
    version = "1.2.0"
)

and main.go:63 passes it to telemetry:

tel, err := telemetry.New(ctx, nodeID, serviceName, commitSHA, version, instanceID)

This PR seeds .release-please-manifest.json with "packages/client-proxy": "0.0.0". On the first release, release-please will therefore produce a tag like client-proxy-v0.0.1 or client-proxy-v0.1.0, and the new publish-client-proxy job in .github/workflows/release-please.yml will push an image tagged v0.0.1 (or similar). That image's telemetry, however, will report version="1.2.0" forever.

Why the existing plumbing doesn't fix it

packages/client-proxy/Makefile build target has -ldflags "-X=main.commitSHA=$(COMMIT_SHA)" — it stamps only commitSHA, not version. And even if the ldflag were added, version is a const (not a var), so -X main.version=... would silently do nothing. packages/client-proxy/Dockerfile accepts ARG COMMIT_SHA but has no ARG VERSION. The new publish-client-proxy job passes --build-arg COMMIT_SHA but no --build-arg VERSION.

Contrast with the sibling pattern this PR copies

The publish-reverse-proxy job in the same workflow does thread the version through end-to-end:

  • packages/docker-reverse-proxy/main.go declares var version = "dev" (a var, stamped via ldflags).
  • Its Makefile has -X=main.version=$(VERSION).
  • Its Dockerfile has ARG VERSION=dev and passes VERSION=${VERSION} to make build.
  • The workflow passes --build-arg VERSION="${IMAGE_TAG}".

The client-proxy publish job omits that last piece and the underlying plumbing was never added.

Step-by-step proof

  1. Someone merges a feat: commit that touches packages/client-proxy/.
  2. release-please opens the release PR bumping packages/client-proxy from 0.0.0 to 0.1.0.
  3. Release PR merges; workflow tags client-proxy-v0.1.0 and triggers publish-client-proxy with VERSION=0.1.0.
  4. The job runs docker buildx build ... --tag us-docker.pkg.dev/e2b-artifacts/client-proxy/client-proxy:v0.1.0 ... (no --build-arg VERSION).
  5. Inside the container, Go binary was compiled with const version = "1.2.0".
  6. Container starts, calls telemetry.New(..., serviceName, commitSHA, "1.2.0", instanceID).
  7. Grafana/Tempo/etc. see service.version=1.2.0 on a v0.1.0 image. Debugging "which release is prod on" gets confusing.

Impact and fix

Telemetry data-quality only — nothing crashes and the proxy works correctly. Two fixes, either works:

  • Cheap: bump .release-please-manifest.json entry for packages/client-proxy to "1.2.0", so the first release-please tag is v1.2.1 / v1.3.0.
  • Aligned with docker-reverse-proxy: flip const version = "1.2.0" to var version = "dev", add -X=main.version=$(VERSION) to the client-proxy Makefile ldflags, add ARG VERSION to packages/client-proxy/Dockerfile, and pass --build-arg VERSION="${IMAGE_TAG}" in publish-client-proxy.

(clickhouse-migrator has no comparable in-binary version constant so it isn't affected.)

"packages/clickhouse": "0.0.0"
}
31 changes: 31 additions & 0 deletions packages/clickhouse/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,24 @@ else
CLICKHOUSE_MIGRATOR_IMAGE ?= $(GCP_REGION)-docker.pkg.dev/$(GCP_PROJECT_ID)/$(PREFIX)core/clickhouse-migrator
endif

# E2B-hosted registry that holds released, prebuilt clickhouse-migrator images.
# Published to by the release-please workflow on tagging a release.
E2B_ARTIFACTS_REGISTRY ?= us-docker.pkg.dev/e2b-artifacts/clickhouse-migrator/clickhouse-migrator

# Optional version "hook" for the released-image flow.
# When empty (default), build-and-upload builds from source and pushes to the
# client's own core repo (the existing flow, unchanged).
# When set (e.g. CLICKHOUSE_MIGRATOR_VERSION=v0.1.0), skip building: pull the
# released image from $(E2B_ARTIFACTS_REGISTRY) and retag/push it into the
# client's core repo, so the Terraform `clickhouse-migrator:latest` lookup
# keeps working.
#
# Set it either on the command line (make ... CLICKHOUSE_MIGRATOR_VERSION=v0.1.0)
# or in the active .env.${ENV} file (CLICKHOUSE_MIGRATOR_VERSION=v0.1.0), which
# is included above. Quotes are stripped so both `=v0.1.0` and `="v0.1.0"` work;
# a command-line value always wins over the env file.
CLICKHOUSE_MIGRATOR_VERSION := $(strip $(subst ",,$(CLICKHOUSE_MIGRATOR_VERSION)))

.PHONY: migrate
migrate: build migrate-without-build

Expand Down Expand Up @@ -37,10 +55,23 @@ build:
@docker build --platform linux/amd64 --tag "$(CLICKHOUSE_MIGRATOR_IMAGE)" --tag "$(CLICKHOUSE_MIGRATOR_IMAGE):$(COMMIT_SHA)" .

.PHONY: build-and-upload
ifeq ($(strip $(CLICKHOUSE_MIGRATOR_VERSION)),)
# Existing flow: build from source and push to the client's own core repo.
build-and-upload:build
$(eval COMMIT_SHA := $(shell git rev-parse --short HEAD))
@docker push "$(CLICKHOUSE_MIGRATOR_IMAGE)"
@docker push "$(CLICKHOUSE_MIGRATOR_IMAGE):$(COMMIT_SHA)"
else
# Released-image flow: pull the prebuilt, versioned image from the E2B
# artifacts registry and retag/push it into the client's own core repo.
build-and-upload:
@echo "Using released clickhouse-migrator $(CLICKHOUSE_MIGRATOR_VERSION) from $(E2B_ARTIFACTS_REGISTRY)"
docker pull --platform linux/amd64 $(E2B_ARTIFACTS_REGISTRY):$(CLICKHOUSE_MIGRATOR_VERSION)
docker tag $(E2B_ARTIFACTS_REGISTRY):$(CLICKHOUSE_MIGRATOR_VERSION) $(CLICKHOUSE_MIGRATOR_IMAGE):latest
docker tag $(E2B_ARTIFACTS_REGISTRY):$(CLICKHOUSE_MIGRATOR_VERSION) $(CLICKHOUSE_MIGRATOR_IMAGE):$(CLICKHOUSE_MIGRATOR_VERSION)
docker push $(CLICKHOUSE_MIGRATOR_IMAGE):latest
docker push $(CLICKHOUSE_MIGRATOR_IMAGE):$(CLICKHOUSE_MIGRATOR_VERSION)
endif

.PHONY: connect-clickhouse
connect-clickhouse:
Expand Down
29 changes: 29 additions & 0 deletions packages/client-proxy/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,23 @@ else
IMAGE_REGISTRY := $(GCP_REGION)-docker.pkg.dev/$(GCP_PROJECT_ID)/$(PREFIX)core/client-proxy
endif

# E2B-hosted registry that holds released, prebuilt client-proxy images.
# Published to by the release-please workflow on tagging a release.
E2B_ARTIFACTS_REGISTRY ?= us-docker.pkg.dev/e2b-artifacts/client-proxy/client-proxy

# Optional version "hook" for the released-image flow.
# When empty (default), build-and-upload builds from source and pushes to the
# client's own core repo (the existing flow, unchanged).
# When set (e.g. CLIENT_PROXY_VERSION=v0.1.0), skip building: pull the released
# image from $(E2B_ARTIFACTS_REGISTRY) and retag/push it into the client's core
# repo, so the Terraform `client-proxy:latest` lookup keeps working.
#
# Set it either on the command line (make ... CLIENT_PROXY_VERSION=v0.1.0) or in
# the active .env.${ENV} file (CLIENT_PROXY_VERSION=v0.1.0), which is included
# above. Quotes are stripped so both `=v0.1.0` and `="v0.1.0"` work; a
# command-line value always wins over the env file.
Comment thread
charlie-e2b marked this conversation as resolved.
CLIENT_PROXY_VERSION := $(strip $(subst ",,$(CLIENT_PROXY_VERSION)))

.PHONY: build
build:
# Allow for passing commit sha directly for docker builds
Expand All @@ -32,8 +49,20 @@ build-debug:

.PHONY: build-and-upload
build-and-upload:
ifeq ($(strip $(CLIENT_PROXY_VERSION)),)
# Existing flow: build from source and push to the client's own core repo.
$(eval COMMIT_SHA := $(shell git rev-parse --short HEAD))
@docker buildx build --platform $(BUILD_PLATFORM) --tag $(IMAGE_REGISTRY) --tag $(IMAGE_REGISTRY):$(COMMIT_SHA) --push --build-arg COMMIT_SHA="$(COMMIT_SHA)" -f ./Dockerfile ..
else
# Released-image flow: pull the prebuilt, versioned image from the E2B
# artifacts registry and retag/push it into the client's own core repo.
@echo "Using released client-proxy $(CLIENT_PROXY_VERSION) from $(E2B_ARTIFACTS_REGISTRY)"
docker pull --platform $(BUILD_PLATFORM) $(E2B_ARTIFACTS_REGISTRY):$(CLIENT_PROXY_VERSION)
docker tag $(E2B_ARTIFACTS_REGISTRY):$(CLIENT_PROXY_VERSION) $(IMAGE_REGISTRY):latest
docker tag $(E2B_ARTIFACTS_REGISTRY):$(CLIENT_PROXY_VERSION) $(IMAGE_REGISTRY):$(CLIENT_PROXY_VERSION)
Comment thread
charlie-e2b marked this conversation as resolved.
docker push $(IMAGE_REGISTRY):latest
docker push $(IMAGE_REGISTRY):$(CLIENT_PROXY_VERSION)
endif
Comment thread
charlie-e2b marked this conversation as resolved.

.PHONY: run
run:
Expand Down
12 changes: 12 additions & 0 deletions release-please-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,18 @@
"component": "docker-reverse-proxy",
"include-v-in-tag": true,
"changelog-path": "CHANGELOG.md"
},
"packages/client-proxy": {
"release-type": "go",
"component": "client-proxy",
"include-v-in-tag": true,
"changelog-path": "CHANGELOG.md"
},
"packages/clickhouse": {
"release-type": "go",
"component": "clickhouse-migrator",
"include-v-in-tag": true,
"changelog-path": "CHANGELOG.md"
}
}
}
Loading