diff --git a/.env.aws.template b/.env.aws.template index b37255d4ce..52055681c4 100644 --- a/.env.aws.template +++ b/.env.aws.template @@ -22,11 +22,13 @@ TERRAFORM_ENVIRONMENT=dev # ------------------------------------------- Optional block ----------------------------------------------------------- # Following variables are optional and doesn't have to be set -# Pull a released, prebuilt docker-reverse-proxy image from E2B's artifact +# Pull a released, prebuilt docker images from E2B's artifact # registry instead of building it from source. When set, `make build-and-upload` # pulls this version and mirrors it into your own core repo (caching it locally). # Leave empty to build from source. Example: v0.1.0 DOCKER_REVERSE_PROXY_VERSION= +CLIENT_PROXY_VERSION= +CLICKHOUSE_MIGRATOR_VERSION= # Sandbox firewall: comma-separated CIDRs to allow through the private-range deny list # ALLOW_SANDBOX_INTERNAL_CIDRS= diff --git a/.env.gcp.template b/.env.gcp.template index 57be039a64..cae3553b96 100644 --- a/.env.gcp.template +++ b/.env.gcp.template @@ -74,11 +74,13 @@ CLICKHOUSE_CLUSTER_SIZE=1 # ------------------------------------------- Optional block ----------------------------------------------------------- # Following variables are optional and doesn't have to be set -# Pull a released, prebuilt docker-reverse-proxy image from E2B's artifact +# Pull a released, prebuilt docker images from E2B's artifact # registry instead of building it from source. When set, `make build-and-upload` # pulls this version and mirrors it into your own core repo (caching it locally). # Leave empty to build from source. Example: v0.1.0 DOCKER_REVERSE_PROXY_VERSION= +CLIENT_PROXY_VERSION= +CLICKHOUSE_MIGRATOR_VERSION= # Dashboard API instance count (default: 0) DASHBOARD_API_COUNT= diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index b4875a8b59..1a1994e594 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -21,6 +21,12 @@ jobs: reverse_proxy_released: ${{ steps.release.outputs['packages/docker-reverse-proxy--release_created'] }} reverse_proxy_tag: ${{ steps.release.outputs['packages/docker-reverse-proxy--tag_name'] }} reverse_proxy_version: ${{ steps.release.outputs['packages/docker-reverse-proxy--version'] }} + client_proxy_released: ${{ steps.release.outputs['packages/client-proxy--release_created'] }} + client_proxy_tag: ${{ steps.release.outputs['packages/client-proxy--tag_name'] }} + client_proxy_version: ${{ steps.release.outputs['packages/client-proxy--version'] }} + clickhouse_migrator_released: ${{ steps.release.outputs['packages/clickhouse--release_created'] }} + clickhouse_migrator_tag: ${{ steps.release.outputs['packages/clickhouse--tag_name'] }} + clickhouse_migrator_version: ${{ steps.release.outputs['packages/clickhouse--version'] }} steps: # Mint a short-lived token from a GitHub App so the release PR is opened as # the App (not the default GITHUB_TOKEN). This is required when `main` has @@ -111,3 +117,107 @@ jobs: --push \ -f packages/docker-reverse-proxy/Dockerfile \ packages + + publish-client-proxy: + name: Publish client-proxy to e2b-artifacts + needs: release-please + if: ${{ needs.release-please.outputs.client_proxy_released == 'true' }} + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + env: + # /// + IMAGE: us-docker.pkg.dev/e2b-artifacts/client-proxy/client-proxy + VERSION: ${{ needs.release-please.outputs.client_proxy_version }} + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Authenticate to the e2b-artifacts project + uses: google-github-actions/auth@v3 + with: + workload_identity_provider: ${{ vars.E2B_ARTIFACTS_WIF_PROVIDER }} + service_account: ${{ vars.E2B_ARTIFACTS_PUBLISH_SA }} + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v3 + + - name: Configure Docker auth + run: gcloud auth configure-docker us-docker.pkg.dev --quiet + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push released image + # The Dockerfile lives in packages/client-proxy but its build context is + # the packages/ directory (it needs shared/ and clickhouse/). Same + # tagging conventions as docker-reverse-proxy: v-prefixed tag, no + # :latest (immutableTags=true on the e2b-artifacts repo), and + # --provenance=false --sbom=false for Artifact Registry compatibility. + run: | + IMAGE_TAG="v${VERSION}" + docker buildx build \ + --platform linux/amd64 \ + --provenance=false \ + --sbom=false \ + --build-arg COMMIT_SHA="${GITHUB_SHA::7}" \ + --tag "${IMAGE}:${IMAGE_TAG}" \ + --push \ + -f packages/client-proxy/Dockerfile \ + packages + + publish-clickhouse-migrator: + name: Publish clickhouse-migrator to e2b-artifacts + needs: release-please + if: ${{ needs.release-please.outputs.clickhouse_migrator_released == 'true' }} + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + env: + # /// + IMAGE: us-docker.pkg.dev/e2b-artifacts/clickhouse-migrator/clickhouse-migrator + VERSION: ${{ needs.release-please.outputs.clickhouse_migrator_version }} + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Authenticate to the e2b-artifacts project + uses: google-github-actions/auth@v3 + with: + workload_identity_provider: ${{ vars.E2B_ARTIFACTS_WIF_PROVIDER }} + service_account: ${{ vars.E2B_ARTIFACTS_PUBLISH_SA }} + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v3 + + - name: Configure Docker auth + run: gcloud auth configure-docker us-docker.pkg.dev --quiet + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push released image + # The clickhouse-migrator image is self-contained: its Dockerfile and + # build context both live in packages/clickhouse (matching the package + # Makefile's `docker build .`). Same tagging conventions as + # docker-reverse-proxy: v-prefixed tag, no :latest (immutableTags=true + # on the e2b-artifacts repo), and --provenance=false --sbom=false for + # Artifact Registry compatibility. + run: | + IMAGE_TAG="v${VERSION}" + docker buildx build \ + --platform linux/amd64 \ + --provenance=false \ + --sbom=false \ + --tag "${IMAGE}:${IMAGE_TAG}" \ + --push \ + -f packages/clickhouse/Dockerfile \ + packages/clickhouse diff --git a/.release-please-manifest.json b/.release-please-manifest.json index d240e2e29b..58d76bed11 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,5 @@ { - "packages/docker-reverse-proxy": "0.2.2" + "packages/docker-reverse-proxy": "0.2.2", + "packages/client-proxy": "0.0.0", + "packages/clickhouse": "0.0.0" } diff --git a/packages/clickhouse/Makefile b/packages/clickhouse/Makefile index a6c4b39d23..e9d132b497 100644 --- a/packages/clickhouse/Makefile +++ b/packages/clickhouse/Makefile @@ -10,6 +10,24 @@ else CLICKHOUSE_MIGRATOR_IMAGE ?= $(GCP_REGION)-docker.pkg.dev/$(GCP_PROJECT_ID)/$(PREFIX)core/clickhouse-migrator endif +# E2B-hosted registry that holds released, prebuilt clickhouse-migrator images. +# Published to by the release-please workflow on tagging a release. +E2B_ARTIFACTS_REGISTRY ?= us-docker.pkg.dev/e2b-artifacts/clickhouse-migrator/clickhouse-migrator + +# Optional version "hook" for the released-image flow. +# When empty (default), build-and-upload builds from source and pushes to the +# client's own core repo (the existing flow, unchanged). +# When set (e.g. CLICKHOUSE_MIGRATOR_VERSION=v0.1.0), skip building: pull the +# released image from $(E2B_ARTIFACTS_REGISTRY) and retag/push it into the +# client's core repo, so the Terraform `clickhouse-migrator:latest` lookup +# keeps working. +# +# Set it either on the command line (make ... CLICKHOUSE_MIGRATOR_VERSION=v0.1.0) +# or in the active .env.${ENV} file (CLICKHOUSE_MIGRATOR_VERSION=v0.1.0), which +# is included above. Quotes are stripped so both `=v0.1.0` and `="v0.1.0"` work; +# a command-line value always wins over the env file. +CLICKHOUSE_MIGRATOR_VERSION := $(strip $(subst ",,$(CLICKHOUSE_MIGRATOR_VERSION))) + .PHONY: migrate migrate: build migrate-without-build @@ -37,10 +55,23 @@ build: @docker build --platform linux/amd64 --tag "$(CLICKHOUSE_MIGRATOR_IMAGE)" --tag "$(CLICKHOUSE_MIGRATOR_IMAGE):$(COMMIT_SHA)" . .PHONY: build-and-upload +ifeq ($(strip $(CLICKHOUSE_MIGRATOR_VERSION)),) +# Existing flow: build from source and push to the client's own core repo. build-and-upload:build $(eval COMMIT_SHA := $(shell git rev-parse --short HEAD)) @docker push "$(CLICKHOUSE_MIGRATOR_IMAGE)" @docker push "$(CLICKHOUSE_MIGRATOR_IMAGE):$(COMMIT_SHA)" +else +# Released-image flow: pull the prebuilt, versioned image from the E2B +# artifacts registry and retag/push it into the client's own core repo. +build-and-upload: + @echo "Using released clickhouse-migrator $(CLICKHOUSE_MIGRATOR_VERSION) from $(E2B_ARTIFACTS_REGISTRY)" + docker pull --platform linux/amd64 $(E2B_ARTIFACTS_REGISTRY):$(CLICKHOUSE_MIGRATOR_VERSION) + docker tag $(E2B_ARTIFACTS_REGISTRY):$(CLICKHOUSE_MIGRATOR_VERSION) $(CLICKHOUSE_MIGRATOR_IMAGE):latest + docker tag $(E2B_ARTIFACTS_REGISTRY):$(CLICKHOUSE_MIGRATOR_VERSION) $(CLICKHOUSE_MIGRATOR_IMAGE):$(CLICKHOUSE_MIGRATOR_VERSION) + docker push $(CLICKHOUSE_MIGRATOR_IMAGE):latest + docker push $(CLICKHOUSE_MIGRATOR_IMAGE):$(CLICKHOUSE_MIGRATOR_VERSION) +endif .PHONY: connect-clickhouse connect-clickhouse: diff --git a/packages/client-proxy/Makefile b/packages/client-proxy/Makefile index 53398334c4..dd3510b122 100644 --- a/packages/client-proxy/Makefile +++ b/packages/client-proxy/Makefile @@ -19,6 +19,23 @@ else IMAGE_REGISTRY := $(GCP_REGION)-docker.pkg.dev/$(GCP_PROJECT_ID)/$(PREFIX)core/client-proxy endif +# E2B-hosted registry that holds released, prebuilt client-proxy images. +# Published to by the release-please workflow on tagging a release. +E2B_ARTIFACTS_REGISTRY ?= us-docker.pkg.dev/e2b-artifacts/client-proxy/client-proxy + +# Optional version "hook" for the released-image flow. +# When empty (default), build-and-upload builds from source and pushes to the +# client's own core repo (the existing flow, unchanged). +# When set (e.g. CLIENT_PROXY_VERSION=v0.1.0), skip building: pull the released +# image from $(E2B_ARTIFACTS_REGISTRY) and retag/push it into the client's core +# repo, so the Terraform `client-proxy:latest` lookup keeps working. +# +# Set it either on the command line (make ... CLIENT_PROXY_VERSION=v0.1.0) or in +# the active .env.${ENV} file (CLIENT_PROXY_VERSION=v0.1.0), which is included +# above. Quotes are stripped so both `=v0.1.0` and `="v0.1.0"` work; a +# command-line value always wins over the env file. +CLIENT_PROXY_VERSION := $(strip $(subst ",,$(CLIENT_PROXY_VERSION))) + .PHONY: build build: # Allow for passing commit sha directly for docker builds @@ -32,8 +49,20 @@ build-debug: .PHONY: build-and-upload build-and-upload: +ifeq ($(strip $(CLIENT_PROXY_VERSION)),) + # Existing flow: build from source and push to the client's own core repo. $(eval COMMIT_SHA := $(shell git rev-parse --short HEAD)) @docker buildx build --platform $(BUILD_PLATFORM) --tag $(IMAGE_REGISTRY) --tag $(IMAGE_REGISTRY):$(COMMIT_SHA) --push --build-arg COMMIT_SHA="$(COMMIT_SHA)" -f ./Dockerfile .. +else + # Released-image flow: pull the prebuilt, versioned image from the E2B + # artifacts registry and retag/push it into the client's own core repo. + @echo "Using released client-proxy $(CLIENT_PROXY_VERSION) from $(E2B_ARTIFACTS_REGISTRY)" + docker pull --platform $(BUILD_PLATFORM) $(E2B_ARTIFACTS_REGISTRY):$(CLIENT_PROXY_VERSION) + docker tag $(E2B_ARTIFACTS_REGISTRY):$(CLIENT_PROXY_VERSION) $(IMAGE_REGISTRY):latest + docker tag $(E2B_ARTIFACTS_REGISTRY):$(CLIENT_PROXY_VERSION) $(IMAGE_REGISTRY):$(CLIENT_PROXY_VERSION) + docker push $(IMAGE_REGISTRY):latest + docker push $(IMAGE_REGISTRY):$(CLIENT_PROXY_VERSION) +endif .PHONY: run run: diff --git a/release-please-config.json b/release-please-config.json index 29d7f12e04..5cdb9f9cf4 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -9,6 +9,18 @@ "component": "docker-reverse-proxy", "include-v-in-tag": true, "changelog-path": "CHANGELOG.md" + }, + "packages/client-proxy": { + "release-type": "go", + "component": "client-proxy", + "include-v-in-tag": true, + "changelog-path": "CHANGELOG.md" + }, + "packages/clickhouse": { + "release-type": "go", + "component": "clickhouse-migrator", + "include-v-in-tag": true, + "changelog-path": "CHANGELOG.md" } } }