diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index b0dbda9..107ae0c 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -3,7 +3,8 @@ name: Release and Update Website Changelog on: push: tags: - - "v*.*.*" + # main.yml creates release tags without a "v" prefix (e.g. 1.1.3) + - "*.*.*" workflow_dispatch: inputs: test_version: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dcbdf25..4c0ffaf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -287,7 +287,7 @@ jobs: strategy: fail-fast: false matrix: - postgres_tag: ['17', '15'] + postgres_tag: ['18', '17', '15'] steps: @@ -332,6 +332,10 @@ jobs: fail-fast: false matrix: include: + - os: ubuntu-22.04 + arch: x86_64 + name: linux + postgres_version: '18' - os: ubuntu-22.04 arch: x86_64 name: linux @@ -340,6 +344,10 @@ jobs: arch: x86_64 name: linux postgres_version: '15' + - os: ubuntu-22.04-arm + arch: arm64 + name: linux + postgres_version: '18' - os: ubuntu-22.04-arm arch: arm64 name: linux @@ -348,6 +356,10 @@ jobs: arch: arm64 name: linux postgres_version: '15' + - os: macos-15 + arch: arm64 + name: macos + postgres_version: '18' - os: macos-15 arch: arm64 name: macos @@ -356,6 +368,10 @@ jobs: arch: arm64 name: macos postgres_version: '15' + - os: macos-15 + arch: x86_64 + name: macos + postgres_version: '18' - os: macos-15 arch: x86_64 name: macos @@ -617,8 +633,8 @@ jobs: [**Expo**](https://www.npmjs.com/package/@sqliteai/sqlite-sync-expo): `npm install @sqliteai/sqlite-sync-expo` [**Android**](https://central.sonatype.com/artifact/ai.sqlite/sync): `ai.sqlite:sync:${{ steps.tag.outputs.version }}` [**Swift**](https://github.com/sqliteai/sqlite-sync#swift-package): [Installation Guide](https://github.com/sqliteai/sqlite-sync#swift-package) - [**Docker (PostgreSQL)**](https://hub.docker.com/r/sqlitecloud/sqlite-sync-postgres): `docker pull sqlitecloud/sqlite-sync-postgres:17` or `:15` - [**Docker (Supabase)**](https://hub.docker.com/r/sqlitecloud/sqlite-sync-supabase): `docker pull sqlitecloud/sqlite-sync-supabase:17`, `:15` or `:17-alpine` + [**Docker (PostgreSQL)**](https://hub.docker.com/r/sqlitecloud/sqlite-sync-postgres): `docker pull sqlitecloud/sqlite-sync-postgres:18`, `:17` or `:15` + [**Docker (Supabase)**](https://hub.docker.com/r/sqlitecloud/sqlite-sync-supabase): `docker pull sqlitecloud/sqlite-sync-supabase:17` or `:15` --- @@ -631,7 +647,7 @@ jobs: docker-publish: runs-on: ubuntu-22.04 - name: docker ${{ matrix.image }} pg${{ matrix.pg_major }}${{ matrix.tag_suffix }}${{ github.ref != 'refs/heads/main' && ' beta' || '' }} + name: docker ${{ matrix.image }} pg${{ matrix.pg_major }}${{ github.ref != 'refs/heads/main' && ' beta' || '' }} needs: [build, postgres-test, postgres-build, release] # main: publish the versioned release tags, only when the release job cut a new # release. Any other branch: release is skipped, publish/overwrite a per-branch @@ -646,29 +662,26 @@ jobs: fail-fast: false matrix: include: + - image: sqlite-sync-postgres + pg_major: '18' + dockerfile: docker/postgresql/Dockerfile.release - image: sqlite-sync-postgres pg_major: '17' dockerfile: docker/postgresql/Dockerfile.release - image: sqlite-sync-postgres pg_major: '15' dockerfile: docker/postgresql/Dockerfile.release + # : tracks whatever base Supabase currently ships for that major. + # Both lines are on an Alpine userland now (still a glibc Nix postgres, + # so the same glibc artifact is installed); there are no family suffixes. - image: sqlite-sync-supabase pg_major: '17' dockerfile: docker/postgresql/Dockerfile.supabase.release - supabase_tag: '17.6.1.071' + supabase_tag: '17.6.1.170' - image: sqlite-sync-supabase pg_major: '15' dockerfile: docker/postgresql/Dockerfile.supabase.release - supabase_tag: '15.8.1.135' - # Newer Supabase 17 images use an Alpine userland (still a glibc Nix - # postgres, so the same glibc artifact is installed). Published behind - # explicit -alpine / exact-base tags; the shared :17 alias stays on the - # Ubuntu build until we decide the alias scheme. - - image: sqlite-sync-supabase - pg_major: '17' - dockerfile: docker/postgresql/Dockerfile.supabase.release - supabase_tag: '17.6.1.151' - tag_suffix: '-alpine' + supabase_tag: '15.14.1.170' steps: @@ -738,15 +751,12 @@ jobs: VERSION=${{ steps.version.outputs.version }} IMAGE=${{ env.DOCKERHUB_ORG }}/${{ matrix.image }} SUPABASE_TAG=${{ matrix.supabase_tag }} - # Empty for the default (Ubuntu) rows; "-alpine" for the Alpine variant. - # Keeps the Alpine images from colliding with the : aliases. - SUFFIX="${{ matrix.tag_suffix }}" BETA="${{ steps.beta.outputs.slug }}" { if [ -n "$BETA" ]; then - echo "tags=${IMAGE}:${{ matrix.pg_major }}${SUFFIX}-beta-${BETA}" + echo "tags=${IMAGE}:${{ matrix.pg_major }}-beta-${BETA}" else - echo "tags=${IMAGE}:${{ matrix.pg_major }}${SUFFIX},${IMAGE}:${{ matrix.pg_major }}${SUFFIX}-${VERSION},${IMAGE}:${SUPABASE_TAG}" + echo "tags=${IMAGE}:${{ matrix.pg_major }},${IMAGE}:${{ matrix.pg_major }}-${VERSION},${IMAGE}:${SUPABASE_TAG}" fi echo "build_args<` is the only tag you need; the transitional `:17-alpine` tag is retired because `:17` is the same image. Exact-base tags (e.g. `:17.6.1.170`) are still published for stacks that pin one. Supabase base images do not pin the `postgres` user's UID, so moving an **existing** data directory across base families needs a one-time `chown` of the data directory and the `supabase_db-config` volume — see [Self-Hosted Supabase](docs/postgresql/quickstarts/supabase-self-hosted.md) for how to discover the new UID and apply it. The CloudSync extension itself is unchanged. - **The local build-from-source path (`make postgres-supabase-build`, `docker/postgresql/Dockerfile.supabase`) now works with the Alpine-userland Supabase bases** as well as the Ubuntu ones. The extension is compiled in a dedicated glibc builder stage (official `postgres:` image with PGDG headers, the same toolchain that produces the release artifacts) whose PostgreSQL major version is derived from the base image tag, and `pg_config` is resolved by probing the Nix profile paths instead of a hardcoded Ubuntu-only path. The runtime stage no longer needs a compiler or package manager, so the base userland is irrelevant. -- **The `sqlitecloud/sqlite-sync-supabase:15` image now builds on Supabase base `15.8.1.135`** (previously `15.8.1.085`), which moves it from Ubuntu 20.04 to 24.04. Ubuntu 24.04 allocates system UIDs differently, so the `postgres` user changes from `105:106` to `101:102` and an **existing** data directory becomes unreadable: the container fails with `cat: /etc/postgresql-custom/pgsodium_root.key: Permission denied` followed by `FATAL: invalid secret key`. Existing deployments need a one-time `chown -R 101:102` on the data directory and the `supabase_db-config` volume before starting the new image — see [Self-Hosted Supabase](docs/postgresql/quickstarts/supabase-self-hosted.md) for the procedure. New deployments are unaffected, and the CloudSync extension itself is unchanged. +- **The development Docker Compose files mount the Postgres data volume at `/var/lib/postgresql` instead of `/var/lib/postgresql/data`.** PostgreSQL 18 keeps its data in a major-version subdirectory and refuses to start when a volume is mounted at the old path; the parent path works for every supported major. Existing local volumes need to be recreated (`docker compose -f docker/postgresql/docker-compose.yml down -v`). + +### Fixed + +- PostgreSQL payload apply now supports wide tables by growing prepared-statement parameter storage dynamically instead of rejecting statements with more than 32 bound values. +- The website changelog workflow now triggers on the release tags this repository actually creates (`1.1.3`); it was listening for `v`-prefixed tags and never fired automatically. ## [1.1.2] - 2026-07-13 diff --git a/docker/Makefile.postgresql b/docker/Makefile.postgresql index c765d8d..ddfc2ce 100644 --- a/docker/Makefile.postgresql +++ b/docker/Makefile.postgresql @@ -221,7 +221,7 @@ DOCKER_TAG ?= latest DOCKER_BUILD_ARGS ?= SUPABASE_CLI_IMAGE ?= $(shell docker ps --format '{{.Image}} {{.Names}}' | awk '/supabase_db/ {print $$1; exit}') SUPABASE_CLI_DOCKERFILE ?= docker/postgresql/Dockerfile.supabase -SUPABASE_POSTGRES_TAG ?= 17.6.1.071 +SUPABASE_POSTGRES_TAG ?= 17.6.1.170 SUPABASE_WORKDIR ?= SUPABASE_WORKDIR_ARG = $(if $(SUPABASE_WORKDIR),--workdir $(SUPABASE_WORKDIR),) SUPABASE_DB_HOST ?= 127.0.0.1 diff --git a/docker/README.md b/docker/README.md index d9e1fa4..95777d6 100644 --- a/docker/README.md +++ b/docker/README.md @@ -31,7 +31,7 @@ docker exec -it cloudsync-postgres psql -U postgres -d cloudsync_test -c "CREATE ``` This starts: -- PostgreSQL 16 on `localhost:5432` +- PostgreSQL 17 on `localhost:5432` (override with `POSTGRES_TAG`, e.g. `18` or `15`) - CloudSync extension pre-installed - pgAdmin on `localhost:5050` (optional, use `--profile admin`) @@ -107,19 +107,18 @@ Run the SQL that exercises the code path. If `psql` blocks, the backend is pause Use this when you're running `supabase start` and want CloudSync inside the local stack. The Supabase CLI uses a bundled PostgreSQL image (for example, -`public.ecr.aws/supabase/postgres:17.6.1.071`). Build a matching image that +`public.ecr.aws/supabase/postgres:17.6.1.170`). Build a matching image that includes CloudSync, then tag it with the same name so the CLI reuses it. This keeps your local Supabase stack intact (auth, realtime, storage, etc.) while enabling the extension in the CLI-managed Postgres container. -> **Note on Ubuntu vs Alpine bases.** Newer Supabase Postgres images (roughly -> `17.6.1.084` and later) use an Alpine-based userland instead of the earlier -> Ubuntu one. The published release image (`Dockerfile.supabase.release`, used -> for `sqlitecloud/sqlite-sync-supabase:*`) detects the userland automatically -> and works on both. The local build-from-source flow below -> (`make postgres-supabase-build`, `Dockerfile.supabase`) currently assumes an -> Ubuntu base; if your CLI stack pulls a newer Alpine image, prefer the -> published `:17-alpine` release image. +> **Note on Ubuntu vs Alpine bases.** Supabase moved both the PG15 and PG17 +> lines to an Alpine-based userland, starting at roughly `15.14.1.x` and +> `17.6.1.084` respectively; earlier bases in both lines were Ubuntu. The published release +> image (`Dockerfile.supabase.release`, used for +> `sqlitecloud/sqlite-sync-supabase:*`) and the local build-from-source flow +> below (`make postgres-supabase-build`, `Dockerfile.supabase`) both detect the +> userland automatically and work on either family. ### Prerequisites @@ -149,12 +148,12 @@ enabling the extension in the CLI-managed Postgres container. `SUPABASE_CLI_IMAGE=public.ecr.aws/supabase/postgres:`. Example: ```bash - SUPABASE_CLI_IMAGE=public.ecr.aws/supabase/postgres:17.6.1.071 make postgres-supabase-build + SUPABASE_CLI_IMAGE=public.ecr.aws/supabase/postgres:17.6.1.170 make postgres-supabase-build ``` You can also set the Supabase base image tag explicitly (defaults to - `17.6.1.071`). This only affects the base image used in the Dockerfile: + `17.6.1.170`). This only affects the base image used in the Dockerfile: ```bash - SUPABASE_POSTGRES_TAG=17.6.1.071 make postgres-supabase-build + SUPABASE_POSTGRES_TAG=17.6.1.170 make postgres-supabase-build ``` 4. Restart the stack: diff --git a/docker/postgresql/Dockerfile.supabase b/docker/postgresql/Dockerfile.supabase index d132aac..1010702 100644 --- a/docker/postgresql/Dockerfile.supabase +++ b/docker/postgresql/Dockerfile.supabase @@ -1,8 +1,8 @@ # Supabase PostgreSQL with CloudSync (sqlite-sync) built from source # # Works with both Supabase base image families: -# - Ubuntu/glibc images (e.g. PG15 15.8.1.135, older PG17 17.6.1.071) -# - Alpine-userland images (e.g. PG17 17.6.1.084+); still a glibc Nix postgres +# - Alpine-userland images, which both the PG15 and PG17 lines now ship +# - the older Ubuntu/glibc images (PG17 up to 17.6.1.071, PG15 up to 15.8.1.x) # Both families run a Nix-built, glibc-linked PostgreSQL, so the extension is # compiled in a glibc builder stage (official postgres image + PGDG headers, # the same toolchain that produces the release artifacts) and the identical @@ -12,7 +12,7 @@ # PG_MAJOR must match the major version of SUPABASE_POSTGRES_TAG; the # postgres-supabase-build Makefile target derives it automatically. ARG PG_MAJOR=17 -ARG SUPABASE_POSTGRES_TAG=17.6.1.071 +ARG SUPABASE_POSTGRES_TAG=17.6.1.170 # Build stage for CloudSync extension (glibc toolchain, matching PG headers) FROM postgres:${PG_MAJOR} AS cloudsync-builder diff --git a/docker/postgresql/Dockerfile.supabase.release b/docker/postgresql/Dockerfile.supabase.release index 6e56b43..22fde84 100644 --- a/docker/postgresql/Dockerfile.supabase.release +++ b/docker/postgresql/Dockerfile.supabase.release @@ -1,15 +1,15 @@ # Supabase PostgreSQL with pre-compiled CloudSync (sqlite-sync) extension # # Works with both Supabase base image families: -# - Ubuntu/glibc images (e.g. PG15 15.8.1.135, older PG17 17.6.1.071) -# - Alpine-userland images (e.g. PG17 17.6.1.084+); still a glibc Nix postgres +# - Alpine-userland images, which both the PG15 and PG17 lines now ship +# - the older Ubuntu/glibc images (PG17 up to 17.6.1.071, PG15 up to 15.8.1.x) # Both run a Nix-built, glibc-linked PostgreSQL, so the SAME glibc extension # artifact is installed in either case. Only the build-time userland tooling # differs (apt-get vs apk), which this file detects at runtime. # # Usage: # docker build \ -# --build-arg SUPABASE_POSTGRES_TAG=17.6.1.151 \ +# --build-arg SUPABASE_POSTGRES_TAG=17.6.1.170 \ # --build-arg CLOUDSYNC_VERSION= \ # -f docker/postgresql/Dockerfile.supabase.release \ # -t my-cloudsync-supabase-postgres . @@ -18,7 +18,7 @@ # docker pull sqlitecloud/sqlite-sync-supabase:17 # -ARG SUPABASE_POSTGRES_TAG=17.6.1.071 +ARG SUPABASE_POSTGRES_TAG=17.6.1.170 FROM public.ecr.aws/supabase/postgres:${SUPABASE_POSTGRES_TAG} ARG CLOUDSYNC_VERSION diff --git a/docker/postgresql/docker-compose.debug.yml b/docker/postgresql/docker-compose.debug.yml index d445670..c5f38f2 100644 --- a/docker/postgresql/docker-compose.debug.yml +++ b/docker/postgresql/docker-compose.debug.yml @@ -23,7 +23,10 @@ services: - ../../Makefile:/tmp/cloudsync/Makefile:ro - ../../.vscode:/tmp/cloudsync/.vscode:ro # Persist database data - - postgres_data:/var/lib/postgresql/data + # PostgreSQL 18+ stores data in a major-version subdirectory and refuses to + # start with a mount at the old /var/lib/postgresql/data path, so mount the + # parent instead — it works for every supported major. + - postgres_data:/var/lib/postgresql # Mount init script - ./init.sql:/docker-entrypoint-initdb.d/init.sql:ro healthcheck: diff --git a/docker/postgresql/docker-compose.yml b/docker/postgresql/docker-compose.yml index 5a3257d..f215daf 100644 --- a/docker/postgresql/docker-compose.yml +++ b/docker/postgresql/docker-compose.yml @@ -18,7 +18,10 @@ services: - ../../docker:/tmp/cloudsync/docker:ro - ../../Makefile:/tmp/cloudsync/Makefile:ro # Persist database data - - postgres_data:/var/lib/postgresql/data + # PostgreSQL 18+ stores data in a major-version subdirectory and refuses to + # start with a mount at the old /var/lib/postgresql/data path, so mount the + # parent instead — it works for every supported major. + - postgres_data:/var/lib/postgresql # Mount init script - ./init.sql:/docker-entrypoint-initdb.d/init.sql:ro healthcheck: diff --git a/docs/internal/supabase-flyio-fly-deploy.md b/docs/internal/supabase-flyio-fly-deploy.md index 14ee49a..a71a631 100644 --- a/docs/internal/supabase-flyio-fly-deploy.md +++ b/docs/internal/supabase-flyio-fly-deploy.md @@ -103,7 +103,7 @@ is intended hardening, not a broken route. ```yaml services: db: - image: ${CLOUDSYNC_IMAGE:-sqlitecloud/sqlite-sync-supabase:17-alpine} + image: ${CLOUDSYNC_IMAGE:-sqlitecloud/sqlite-sync-supabase:17} volumes: - ./cloudsync.sql:/docker-entrypoint-initdb.d/init-scripts/100-cloudsync.sql:Z ports: @@ -140,7 +140,7 @@ override: ```bash fly secrets set -a "$FLY_APP" \ - CLOUDSYNC_IMAGE=sqlitecloud/sqlite-sync-supabase:17-alpine-beta- + CLOUDSYNC_IMAGE=sqlitecloud/sqlite-sync-supabase:17-beta- ``` ### Secrets @@ -208,7 +208,7 @@ release the public IPs, and use `fly proxy 8000 -a `. | Task | Command | |---|---| -| Update the CloudSync image | released tag: redeploy (compose pulls `:17-alpine`); beta: `fly secrets set CLOUDSYNC_IMAGE=…` | +| Update the CloudSync image | released tag: redeploy (compose pulls `:17`); beta: `fly secrets set CLOUDSYNC_IMAGE=…` | | Update Supabase services | bump `SUPABASE_REF` in the `Dockerfile`, `fly deploy` | | Restart the stack | `fly machine restart -a ` — the entrypoint re-runs everything | | Inspect containers | `fly ssh console -a -C "docker ps"` | @@ -380,7 +380,7 @@ it with `CLOUDSYNC_IMAGE`. To build one locally instead: git submodule update --init --recursive # else: fractional_indexing.h: No such file or directory docker build --platform linux/amd64 \ - --build-arg SUPABASE_POSTGRES_TAG=17.6.1.151 \ + --build-arg SUPABASE_POSTGRES_TAG=17.6.1.170 \ -f docker/postgresql/Dockerfile.supabase \ -t /: . @@ -415,10 +415,11 @@ without credentials, or `docker login` has to run on the Machine. ## Image / version notes -The Alpine CloudSync images are built from `supabase/postgres:17.6.1.151` (see the -publish matrix in `.github/workflows/main.yml`), while the pinned upstream compose -expects `17.6.1.136`. Same PG 17 line, newer patch base — fine, but re-check after -bumping either side. +`:17` is built from `supabase/postgres:17.6.1.170` (see the publish matrix in +`.github/workflows/main.yml`), while the pinned upstream compose expects +`17.6.1.136`. Same PG 17 line, newer patch base — fine, but re-check after bumping +either side. The tag tracks whatever base Supabase currently ships for the major, +so it moves when upstream does; there is no longer a separate `:17-alpine` tag. The pinned compose runs 11 services: `db`, `supavisor`, `auth`, `rest`, `realtime`, `storage`, `imgproxy`, `meta`, `functions`, `studio` and the gateway. Upstream has diff --git a/docs/postgresql/quickstarts/postgres.md b/docs/postgresql/quickstarts/postgres.md index fcfefe8..ec29a3b 100644 --- a/docs/postgresql/quickstarts/postgres.md +++ b/docs/postgresql/quickstarts/postgres.md @@ -13,13 +13,14 @@ You can enable CloudSync in one of two ways: Use the published PostgreSQL image that already includes the CloudSync extension: - `sqlitecloud/sqlite-sync-postgres:15` - `sqlitecloud/sqlite-sync-postgres:17` +- `sqlitecloud/sqlite-sync-postgres:18` Example using Docker Compose: ```yaml services: db: - image: sqlitecloud/sqlite-sync-postgres:17 + image: sqlitecloud/sqlite-sync-postgres:18 container_name: cloudsync-postgres environment: POSTGRES_USER: postgres @@ -28,13 +29,15 @@ services: ports: - "5432:5432" volumes: - - pg_data:/var/lib/postgresql/data + - pg_data:/var/lib/postgresql - ./init.sql:/docker-entrypoint-initdb.d/init.sql:ro volumes: pg_data: ``` +**Data volume path:** PostgreSQL 18 stores its data in a major-version subdirectory and refuses to start if a volume is mounted at the old `/var/lib/postgresql/data` path, so mount the parent `/var/lib/postgresql` as above. That path also works on 15 and 17, which keep their data in `data/` underneath it. An existing 15 or 17 volume mounted at `/var/lib/postgresql/data` is unaffected until you change the mount, and moving between major versions still requires `pg_upgrade`. + Create `init.sql`: ```sql CREATE EXTENSION IF NOT EXISTS cloudsync; diff --git a/docs/postgresql/quickstarts/supabase-self-hosted.md b/docs/postgresql/quickstarts/supabase-self-hosted.md index 0e1c910..a19f40f 100644 --- a/docs/postgresql/quickstarts/supabase-self-hosted.md +++ b/docs/postgresql/quickstarts/supabase-self-hosted.md @@ -14,20 +14,18 @@ Follow [Supabase's Installing Supabase](https://supabase.com/docs/guides/self-ho db: # Supabase on PostgreSQL 15 image: sqlitecloud/sqlite-sync-supabase:15 - # instead of: public.ecr.aws/supabase/postgres:15.8.1.135 + # instead of: public.ecr.aws/supabase/postgres:15.14.1.170 # OR Supabase on PostgreSQL 17 image: sqlitecloud/sqlite-sync-supabase:17 - # instead of: public.ecr.aws/supabase/postgres:17.6.1.071 - - # OR Supabase on PostgreSQL 17 with a newer Alpine-based Supabase image - image: sqlitecloud/sqlite-sync-supabase:17-alpine - # instead of: public.ecr.aws/supabase/postgres:17.6.1.151 + # instead of: public.ecr.aws/supabase/postgres:17.6.1.170 ``` -Use the CloudSync image tag that matches your Supabase PostgreSQL major version. The published major tags `sqlitecloud/sqlite-sync-supabase:15` and `sqlitecloud/sqlite-sync-supabase:17` are the standard choice. +Use the CloudSync image tag that matches your Supabase PostgreSQL major version. `:15` and `:17` each track the base image Supabase currently ships for that major, so they are the standard choice. + +**Pinning an exact base:** if your Supabase stack pins a specific base image, an exact-base CloudSync tag is published alongside the major tags (e.g. `sqlitecloud/sqlite-sync-supabase:17.6.1.170`). Use it when you need the CloudSync image built from precisely the base your stack expects; otherwise the major tag is simpler. -**Ubuntu vs Alpine base:** newer Supabase Postgres images (roughly `17.6.1.084` and later) ship an Alpine-based userland instead of the earlier Ubuntu one. The extension itself is identical on both — the PostgreSQL binary is glibc-linked in either case — so for a new deployment the two are interchangeable. Moving an **existing** deployment from one family to the other needs the ownership fix described under [For Existing Deployments](#for-existing-deployments). If your Supabase stack pins one of these newer Alpine base images, use the matching `sqlitecloud/sqlite-sync-supabase:17-alpine` tag (or the exact base tag, e.g. `sqlitecloud/sqlite-sync-supabase:17.6.1.151`) so the CloudSync image is built from the same base. Exact Supabase base-image tags are published for both families but are optional for normal setup. +**Ubuntu vs Alpine base:** Supabase moved both the PG15 and PG17 lines to an Alpine userland — starting at roughly `15.14.1.x` and `17.6.1.084` respectively; earlier bases in both lines were Ubuntu. The extension is identical on either — the PostgreSQL binary is glibc-linked in both families — so for a new deployment the difference does not matter. Moving an **existing** data directory from one family to the other needs the ownership fix described under [For Existing Deployments](#for-existing-deployments). ### Add the CloudSync Init Script @@ -67,7 +65,6 @@ Follow [Supabase's Updating](https://supabase.com/docs/guides/self-hosting/docke # Update docker-compose.yml to use: # sqlitecloud/sqlite-sync-supabase:15 # or sqlitecloud/sqlite-sync-supabase:17 -# or sqlitecloud/sqlite-sync-supabase:17-alpine (newer Alpine-based Supabase bases) docker compose pull docker compose down && docker compose up -d ``` @@ -80,7 +77,7 @@ CREATE EXTENSION IF NOT EXISTS cloudsync; #### If Postgres fails to start after moving to a different base image -Supabase base images do not pin the `postgres` user's UID, so it differs between them — for example `105` on the Ubuntu 20.04 based `15.8.1.085`, `101` on the Ubuntu 24.04 based `15.8.1.135`, and `100` on the Alpine based `17.6.1.151`. A data directory created under one base is unreadable by another, and the container fails to start with: +Supabase base images do not pin the `postgres` user's UID, so it differs between them — for example `105` on the Ubuntu 20.04 based `15.8.1.085`, `101` on the Ubuntu 24.04 based `15.8.1.135`, and `100` on the Alpine based `17.6.1.151`. Discover the UID of the image you are moving to rather than assuming one. A data directory created under one base is unreadable by another, and the container fails to start with: ``` cat: /etc/postgresql-custom/pgsodium_root.key: Permission denied diff --git a/src/cloudsync.h b/src/cloudsync.h index 3c2340d..7e72f86 100644 --- a/src/cloudsync.h +++ b/src/cloudsync.h @@ -18,7 +18,7 @@ extern "C" { #endif -#define CLOUDSYNC_VERSION "1.1.2" +#define CLOUDSYNC_VERSION "1.1.3" #define CLOUDSYNC_MAX_TABLENAME_LEN 512 #define CLOUDSYNC_VALUE_NOTSET -1 diff --git a/src/postgresql/database_postgresql.c b/src/postgresql/database_postgresql.c index 3fc6310..0f9a50b 100644 --- a/src/postgresql/database_postgresql.c +++ b/src/postgresql/database_postgresql.c @@ -10,6 +10,7 @@ #include "postgres.h" #include +#include #include #include @@ -49,7 +50,7 @@ // PostgreSQL SPI handles require knowing parameter count and types upfront. // Solution: Defer actual SPI_prepare until first step(), after all bindings are set. -#define MAX_PARAMS 32 +#define INITIAL_PARAM_CAPACITY 32 typedef struct { // Prepared plan @@ -67,11 +68,12 @@ typedef struct { // Params int nparams; - Oid types[MAX_PARAMS]; - Oid prepared_types[MAX_PARAMS]; // types used when plan was SPI_prepare'd + int param_capacity; + Oid *types; + Oid *prepared_types; // types used when plan was SPI_prepare'd int prepared_nparams; // nparams at prepare time - Datum values[MAX_PARAMS]; - char nulls[MAX_PARAMS]; + Datum *values; + char *nulls; bool executed_nonselect; // non-select executed already // Memory @@ -85,6 +87,7 @@ typedef struct { } pg_stmt_t; static int database_refresh_snapshot (void); +static int databasevm_reserve_params (pg_stmt_t *stmt, int required); // MARK: - SQL - @@ -1988,6 +1991,64 @@ int database_pk_names (cloudsync_context *data, const char *table_name, char *** // MARK: - VM - +static int databasevm_reserve_params (pg_stmt_t *stmt, int required) { + if (!stmt || required < 0) return DBRES_ERROR; + if (required <= stmt->param_capacity) return DBRES_OK; + + int new_capacity = stmt->param_capacity > 0 ? stmt->param_capacity : INITIAL_PARAM_CAPACITY; + while (new_capacity < required) { + if (new_capacity > INT_MAX / 2) { + new_capacity = required; + break; + } + new_capacity *= 2; + } + + if ((Size)new_capacity > MaxAllocSize / sizeof(Oid) || + (Size)new_capacity > MaxAllocSize / sizeof(Datum)) { + return cloudsync_set_error(stmt->data, "Too many SQL parameters", DBRES_NOMEM); + } + + Oid *new_types = (Oid *)MemoryContextAllocExtended( + stmt->stmt_mcxt, sizeof(Oid) * (Size)new_capacity, MCXT_ALLOC_ZERO | MCXT_ALLOC_NO_OOM); + Oid *new_prepared_types = (Oid *)MemoryContextAllocExtended( + stmt->stmt_mcxt, sizeof(Oid) * (Size)new_capacity, MCXT_ALLOC_ZERO | MCXT_ALLOC_NO_OOM); + Datum *new_values = (Datum *)MemoryContextAllocExtended( + stmt->stmt_mcxt, sizeof(Datum) * (Size)new_capacity, MCXT_ALLOC_ZERO | MCXT_ALLOC_NO_OOM); + char *new_nulls = (char *)MemoryContextAllocExtended( + stmt->stmt_mcxt, sizeof(char) * (Size)new_capacity, MCXT_ALLOC_ZERO | MCXT_ALLOC_NO_OOM); + + if (!new_types || !new_prepared_types || !new_values || !new_nulls) { + if (new_types) pfree(new_types); + if (new_prepared_types) pfree(new_prepared_types); + if (new_values) pfree(new_values); + if (new_nulls) pfree(new_nulls); + return cloudsync_set_error(stmt->data, "Not enough memory for SQL parameters", DBRES_NOMEM); + } + + if (stmt->param_capacity > 0) { + memcpy(new_types, stmt->types, sizeof(Oid) * (Size)stmt->param_capacity); + memcpy(new_prepared_types, stmt->prepared_types, sizeof(Oid) * (Size)stmt->param_capacity); + memcpy(new_values, stmt->values, sizeof(Datum) * (Size)stmt->param_capacity); + memcpy(new_nulls, stmt->nulls, sizeof(char) * (Size)stmt->param_capacity); + } + for (int i = stmt->param_capacity; i < new_capacity; i++) { + new_types[i] = UNKNOWNOID; + new_nulls[i] = 'n'; + } + + if (stmt->types) pfree(stmt->types); + if (stmt->prepared_types) pfree(stmt->prepared_types); + if (stmt->values) pfree(stmt->values); + if (stmt->nulls) pfree(stmt->nulls); + stmt->types = new_types; + stmt->prepared_types = new_prepared_types; + stmt->values = new_values; + stmt->nulls = new_nulls; + stmt->param_capacity = new_capacity; + return DBRES_OK; +} + int databasevm_prepare (cloudsync_context *data, const char *sql, dbvm_t **vm, int flags) { if (!sql || !vm) { return cloudsync_set_error(data, "Invalid parameters to databasevm_prepare", DBRES_ERROR); @@ -1995,10 +2056,6 @@ int databasevm_prepare (cloudsync_context *data, const char *sql, dbvm_t **vm, i *vm = NULL; cloudsync_reset_error(data); - // sanity check number of parameters - // int counter = count_params(sql); - // if (counter > MAX_PARAMS) return cloudsync_set_error(data, "Maximum number of parameters reached", DBRES_MISUSE); - // create PostgreSQL VM statement pg_stmt_t *stmt = (pg_stmt_t *)cloudsync_memory_zeroalloc(sizeof(pg_stmt_t)); if (!stmt) return cloudsync_set_error(data, "Not enough memory to allocate a dbvm_t struct", DBRES_NOMEM); @@ -2065,7 +2122,9 @@ int databasevm_step0 (pg_stmt_t *stmt) { stmt->plan_is_prepared = true; // Save the types used for this plan so we can detect type changes - memcpy(stmt->prepared_types, stmt->types, sizeof(Oid) * stmt->nparams); + if (stmt->nparams > 0) { + memcpy(stmt->prepared_types, stmt->types, sizeof(Oid) * (Size)stmt->nparams); + } stmt->prepared_nparams = stmt->nparams; } PG_CATCH(); @@ -2321,7 +2380,7 @@ void databasevm_clear_bindings (dbvm_t *vm) { stmt->nparams = 0; // Reset params array to defaults - for (int i = 0; i < MAX_PARAMS; i++) { + for (int i = 0; i < stmt->param_capacity; i++) { stmt->types[i] = UNKNOWNOID; stmt->values[i] = (Datum) 0; stmt->nulls[i] = 'n'; // default NULL @@ -2355,9 +2414,9 @@ int databasevm_bind_blob (dbvm_t *vm, int index, const void *value, uint64_t siz if (size > (uint64) (MaxAllocSize - VARHDRSZ)) return DBRES_NOMEM; int idx = index - 1; - if (idx >= MAX_PARAMS) return DBRES_ERROR; - pg_stmt_t *stmt = (pg_stmt_t*)vm; + int rc = databasevm_reserve_params(stmt, index); + if (rc != DBRES_OK) return rc; MemoryContext old = MemoryContextSwitchTo(stmt->bind_mcxt); // Convert binary data to PostgreSQL bytea @@ -2379,9 +2438,9 @@ int databasevm_bind_double (dbvm_t *vm, int index, double value) { if (!vm || index < 1) return DBRES_ERROR; int idx = index - 1; - if (idx >= MAX_PARAMS) return DBRES_ERROR; - pg_stmt_t *stmt = (pg_stmt_t*)vm; + int rc = databasevm_reserve_params(stmt, index); + if (rc != DBRES_OK) return rc; stmt->values[idx] = Float8GetDatum(value); stmt->types[idx] = FLOAT8OID; stmt->nulls[idx] = ' '; @@ -2394,9 +2453,9 @@ int databasevm_bind_int (dbvm_t *vm, int index, int64_t value) { if (!vm || index < 1) return DBRES_ERROR; int idx = index - 1; - if (idx >= MAX_PARAMS) return DBRES_ERROR; - pg_stmt_t *stmt = (pg_stmt_t*)vm; + int rc = databasevm_reserve_params(stmt, index); + if (rc != DBRES_OK) return rc; stmt->values[idx] = Int64GetDatum(value); stmt->types[idx] = INT8OID; stmt->nulls[idx] = ' '; @@ -2409,9 +2468,9 @@ int databasevm_bind_null (dbvm_t *vm, int index) { if (!vm || index < 1) return DBRES_ERROR; int idx = index - 1; - if (idx >= MAX_PARAMS) return DBRES_ERROR; - pg_stmt_t *stmt = (pg_stmt_t*)vm; + int rc = databasevm_reserve_params(stmt, index); + if (rc != DBRES_OK) return rc; stmt->values[idx] = (Datum)0; stmt->types[idx] = TEXTOID; // TEXTOID has casts to most types stmt->nulls[idx] = 'n'; @@ -2429,9 +2488,9 @@ int databasevm_bind_text (dbvm_t *vm, int index, const char *value, int size) { if ((Size)size > MaxAllocSize - VARHDRSZ) return DBRES_NOMEM; int idx = index - 1; - if (idx >= MAX_PARAMS) return DBRES_ERROR; - pg_stmt_t *stmt = (pg_stmt_t*)vm; + int rc = databasevm_reserve_params(stmt, index); + if (rc != DBRES_OK) return rc; MemoryContext old = MemoryContextSwitchTo(stmt->bind_mcxt); text *t = cstring_to_text_with_len(value, size); @@ -2452,9 +2511,9 @@ int databasevm_bind_value (dbvm_t *vm, int index, dbvalue_t *value) { // validate index bounds properly (1-based index) if (index < 1) return DBRES_ERROR; int idx = index - 1; - if (idx >= MAX_PARAMS) return DBRES_ERROR; - pg_stmt_t *stmt = (pg_stmt_t*)vm; + int rc = databasevm_reserve_params(stmt, index); + if (rc != DBRES_OK) return rc; pgvalue_t *v = (pgvalue_t *)value; if (!v || v->isnull) { stmt->values[idx] = (Datum)0; @@ -3072,5 +3131,3 @@ uint64_t dbmem_size (void *ptr) { // Return 0 as a safe default return 0; } - - diff --git a/test/postgresql/56_many_columns.sql b/test/postgresql/56_many_columns.sql new file mode 100644 index 0000000..aa27649 --- /dev/null +++ b/test/postgresql/56_many_columns.sql @@ -0,0 +1,114 @@ +-- Wide-table payload apply. +-- +-- Exercises the old fixed 32-parameter boundary (32 data columns + one PK) +-- and multiple dynamic parameter-array growths (127 data columns + one PK). +-- Both INSERT/upsert and UPDATE paths must reproduce every column. + +\set testid '56-many-columns' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_56_many_columns_src; +DROP DATABASE IF EXISTS cloudsync_test_56_many_columns_dst; +CREATE DATABASE cloudsync_test_56_many_columns_src; +CREATE DATABASE cloudsync_test_56_many_columns_dst; + +\connect cloudsync_test_56_many_columns_src +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +SELECT 'CREATE TABLE wide_33 (id TEXT PRIMARY KEY' || + string_agg(format(', col%s TEXT', i), '' ORDER BY i) || ');' +FROM generate_series(1, 32) AS g(i) \gexec +SELECT 'CREATE TABLE wide_128 (id TEXT PRIMARY KEY' || + string_agg(format(', col%s TEXT', i), '' ORDER BY i) || ');' +FROM generate_series(1, 127) AS g(i) \gexec + +SELECT cloudsync_init('wide_33', 'CLS', 1) AS _init_33 \gset +SELECT cloudsync_init('wide_128', 'CLS', 1) AS _init_128 \gset + +SELECT 'INSERT INTO wide_33 VALUES (' || quote_literal('row-33') || ', ' || + string_agg(quote_literal('v' || i), ', ' ORDER BY i) || ');' +FROM generate_series(1, 32) AS g(i) \gexec +SELECT 'INSERT INTO wide_128 VALUES (' || quote_literal('row-128') || ', ' || + string_agg(quote_literal('v' || i), ', ' ORDER BY i) || ');' +FROM generate_series(1, 127) AS g(i) \gexec + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, + db_version, site_id, cl, seq), 'hex') AS insert_payload +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_56_many_columns_dst +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; + +SELECT 'CREATE TABLE wide_33 (id TEXT PRIMARY KEY' || + string_agg(format(', col%s TEXT', i), '' ORDER BY i) || ');' +FROM generate_series(1, 32) AS g(i) \gexec +SELECT 'CREATE TABLE wide_128 (id TEXT PRIMARY KEY' || + string_agg(format(', col%s TEXT', i), '' ORDER BY i) || ');' +FROM generate_series(1, 127) AS g(i) \gexec + +SELECT cloudsync_init('wide_33', 'CLS', 1) AS _init_dst_33 \gset +SELECT cloudsync_init('wide_128', 'CLS', 1) AS _init_dst_128 \gset +SELECT cloudsync_payload_apply(decode(:'insert_payload', 'hex')) AS insert_apply_count \gset + +SELECT coalesce(count(*) = 32 AND bool_and(value = 'v' || substring(key FROM 4)), false) AS insert_33_ok +FROM wide_33 AS w +CROSS JOIN LATERAL jsonb_each_text(to_jsonb(w) - 'id') +WHERE id = 'row-33' \gset +SELECT coalesce(count(*) = 127 AND bool_and(value = 'v' || substring(key FROM 4)), false) AS insert_128_ok +FROM wide_128 AS w +CROSS JOIN LATERAL jsonb_each_text(to_jsonb(w) - 'id') +WHERE id = 'row-128' \gset + +SELECT (:insert_apply_count::int = 159 AND :'insert_33_ok'::boolean AND :'insert_128_ok'::boolean) AS insert_ok \gset +\if :insert_ok +\echo [PASS] (:testid) wide INSERT apply crossed the old 32-parameter limit (:insert_apply_count changes) +\else +\echo [FAIL] (:testid) wide INSERT apply failed (count=:insert_apply_count, 33=:insert_33_ok, 128=:insert_128_ok) +SELECT (:fail::int + 1) AS fail \gset +\endif + +\connect cloudsync_test_56_many_columns_src +SELECT 'UPDATE wide_33 SET ' || + string_agg(format('col%s = %L', i, 'u' || i), ', ' ORDER BY i) || + ' WHERE id = ''row-33'';' +FROM generate_series(1, 32) AS g(i) \gexec +SELECT 'UPDATE wide_128 SET ' || + string_agg(format('col%s = %L', i, 'u' || i), ', ' ORDER BY i) || + ' WHERE id = ''row-128'';' +FROM generate_series(1, 127) AS g(i) \gexec + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, + db_version, site_id, cl, seq), 'hex') AS update_payload +FROM cloudsync_changes +WHERE site_id = cloudsync_siteid() \gset + +\connect cloudsync_test_56_many_columns_dst +SELECT cloudsync_payload_apply(decode(:'update_payload', 'hex')) AS update_apply_count \gset + +SELECT coalesce(count(*) = 32 AND bool_and(value = 'u' || substring(key FROM 4)), false) AS update_33_ok +FROM wide_33 AS w +CROSS JOIN LATERAL jsonb_each_text(to_jsonb(w) - 'id') +WHERE id = 'row-33' \gset +SELECT coalesce(count(*) = 127 AND bool_and(value = 'u' || substring(key FROM 4)), false) AS update_128_ok +FROM wide_128 AS w +CROSS JOIN LATERAL jsonb_each_text(to_jsonb(w) - 'id') +WHERE id = 'row-128' \gset + +SELECT (:update_apply_count::int = 159 AND :'update_33_ok'::boolean AND :'update_128_ok'::boolean) AS update_ok \gset +\if :update_ok +\echo [PASS] (:testid) wide UPDATE apply preserved every column (:update_apply_count changes) +\else +\echo [FAIL] (:testid) wide UPDATE apply failed (count=:update_apply_count, 33=:update_33_ok, 128=:update_128_ok) +SELECT (:fail::int + 1) AS fail \gset +\endif + +\ir helper_test_cleanup.sql +\if :should_cleanup +DROP DATABASE IF EXISTS cloudsync_test_56_many_columns_src; +DROP DATABASE IF EXISTS cloudsync_test_56_many_columns_dst; +\endif diff --git a/test/postgresql/full_test.sql b/test/postgresql/full_test.sql index c6f38b7..fc76082 100644 --- a/test/postgresql/full_test.sql +++ b/test/postgresql/full_test.sql @@ -63,6 +63,7 @@ \ir 53_payload_blob_checked_pg_try.sql \ir 54_payload_chunks_fragment_state.sql \ir 55_payload_chunks_positional_resume.sql +\ir 56_many_columns.sql -- 'Test summary' \echo '\nTest summary:'