feat: port the memory engine into a TinyBus module

.github/workflows/ci.yml

@@ -49,6 +49,71 @@ jobs:
       - name: Test default features
         run: cargo test
 
+  # The module crate is its own workspace root (see the `exclude` note in the
+  # root Cargo.toml), so NONE of the steps above touch it: `--all-targets`,
+  # `--all-features` and `--workspace` all stop at the workspace boundary and
+  # exit 0 without having compiled a line of it.
+  #
+  # That silence is the hazard. A cdylib that fails to build is a release that
+  # cannot be cut, and it would be discovered at release time rather than on the
+  # PR that broke it. So it gets its own job with the same gates.
+  module:
+    name: Module (own workspace)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+          submodules: recursive
+
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt, clippy
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Check formatting
+        run: cargo fmt --manifest-path crates/tinymemory-module/Cargo.toml --all -- --check
+
+      - name: Clippy
+        run: >-
+          cargo clippy --manifest-path crates/tinymemory-module/Cargo.toml
+          --all-targets -- -D warnings
+
+      - name: Build the cdylib
+        run: cargo build --manifest-path crates/tinymemory-module/Cargo.toml --release
+
+      - name: Unit tests
+        run: cargo test --manifest-path crates/tinymemory-module/Cargo.toml --lib
+
+      # The loader E2E drives a real dlopen'ed module, and tinybus binds its
+      # broker tasks to the runtime that created them. The module is loaded once
+      # per process and never unloaded, so two such tests in one process leave the
+      # second talking to a dead broker and it HANGS rather than failing. Hence
+      # one process per test, with a timeout so a hang is a red build and not a
+      # six-hour job.
+      - name: Loader E2E (one process per test)
+        env:
+          TINYMEMORY_TEST_MODULE: >-
+            ${{ github.workspace }}/crates/tinymemory-module/target/release/libtinymemory_module.so
+        run: |
+          set -euo pipefail
+          tests=$(
+            cargo test --manifest-path crates/tinymemory-module/Cargo.toml \
+              --test module_e2e -- --ignored --list \
+              | sed -n 's/^\(.*\): test$/\1/p'
+          )
+          if [ -z "$tests" ]; then
+            echo "No ignored E2E tests were found — the list step is broken." >&2
+            exit 1
+          fi
+          for test in $tests; do
+            echo "::group::$test"
+            timeout 300 cargo test --manifest-path crates/tinymemory-module/Cargo.toml \
+              --test module_e2e -- --ignored --exact "$test"
+            echo "::endgroup::"
+          done
+
   docs:
     name: Docs
     runs-on: ubuntu-latest

.github/workflows/release.yml

@@ -11,6 +11,13 @@ on:
           - patch
           - minor
           - major
+      existing_tag:
+        description: >-
+          Re-cut module artifacts for a tag that already exists, skipping the
+          version bump and tag. Leave empty for a normal release.
+        type: string
+        required: false
+        default: ""
 
 concurrency:
   group: release-${{ github.ref_name }}
@@ -20,9 +27,24 @@ permissions:
   contents: write
 
 jobs:
-  publish:
-    name: Publish crate
+  # Cuts the version bump and the tag. It deliberately does **not** publish to
+  # crates.io, and that is not an omission to be fixed later: `tinymemory-core`
+  # depends on `tinycortex-api`, which is consumed by path and is not on
+  # crates.io, so `cargo package` cannot resolve it (`no matching package named
+  # 'tinycortex-api' found`). Every consumer takes this repo by path or git.
+  # The crates are `publish = false` so the two facts cannot drift apart.
+  #
+  # What a release produces is the tag plus the per-platform module archives and
+  # their `checksum.toml` — that is what a host pins and verifies.
+  tag:
+    name: Tag release
     if: ${{ github.ref == 'refs/heads/main' }}
+    # Consumed by `release-target`. The `version` step already writes both to
+    # `$GITHUB_OUTPUT`; without this block they stop at the job boundary and the
+    # module bundles resolve an empty tag.
+    outputs:
+      tag: ${{ steps.version.outputs.tag }}
+      next_version: ${{ steps.version.outputs.next_version }}
     runs-on: ubuntu-latest
     environment: Production
     steps:
@@ -121,9 +143,6 @@ jobs:
           git commit -m "Release ${RELEASE_TAG}"
           git tag -a "${RELEASE_TAG}" -m "Release ${RELEASE_TAG}"
 
-      - name: Package crate
-        run: cargo package --locked
-
       - name: Push release commit and tag
         env:
           RELEASE_TAG: ${{ steps.version.outputs.tag }}
@@ -132,7 +151,255 @@ jobs:
           git push origin "HEAD:${GITHUB_REF_NAME}"
           git push origin "${RELEASE_TAG}"
 
-      - name: Publish to crates.io
-        run: cargo publish --locked
+  release-target:
+    name: Resolve release target
+    needs: tag
+    # `always()` so a skipped tag job still yields a target: re-cutting the
+    # module artifacts for an existing tag is a genuinely independent release.
+    if: ${{ always() && (inputs.existing_tag != '' || needs.tag.result == 'success') }}
+    runs-on: ubuntu-latest
+    outputs:
+      tag: ${{ steps.resolve.outputs.tag }}
+      next_version: ${{ steps.resolve.outputs.next_version }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Resolve the tag and version to build
+        id: resolve
+        shell: bash
+        env:
+          EXISTING_TAG: ${{ inputs.existing_tag }}
+          PUBLISHED_TAG: ${{ needs.tag.outputs.tag }}
+          PUBLISHED_VERSION: ${{ needs.tag.outputs.next_version }}
+        run: |
+          set -euo pipefail
+          if [[ -n "$EXISTING_TAG" ]]; then
+            git fetch --tags origin
+            git rev-parse --verify --quiet "refs/tags/${EXISTING_TAG}" >/dev/null \
+              || { echo "tag ${EXISTING_TAG} does not exist" >&2; exit 1; }
+            tag="$EXISTING_TAG"
+            version="${EXISTING_TAG#v}"
+          else
+            tag="$PUBLISHED_TAG"
+            version="$PUBLISHED_VERSION"
+          fi
+          [[ -n "$tag" && -n "$version" ]] || { echo "could not resolve a release target" >&2; exit 1; }
+          {
+            echo "tag=${tag}"
+            echo "next_version=${version}"
+          } >> "$GITHUB_OUTPUT"
+
+  native-bundles:
+    name: Module bundle (${{ matrix.id }})
+    needs: release-target
+    # `always()` is required even though `release-target` succeeds: GitHub
+    # propagates a skip transitively, so a skipped `tag` upstream would skip
+    # this job regardless of its direct dependency's result. The explicit
+    # success check is what actually gates it.
+    if: ${{ always() && needs.release-target.result == 'success' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - id: ubuntu-22.04-x86_64
+            os: ubuntu-22.04
+            target: x86_64-unknown-linux-gnu
+          - id: ubuntu-22.04-arm64
+            os: ubuntu-22.04-arm
+            target: aarch64-unknown-linux-gnu
+          - id: ubuntu-24.04-x86_64
+            os: ubuntu-24.04
+            target: x86_64-unknown-linux-gnu
+          - id: ubuntu-24.04-arm64
+            os: ubuntu-24.04-arm
+            target: aarch64-unknown-linux-gnu
+          - id: macos-15-x86_64
+            os: macos-15-intel
+            target: x86_64-apple-darwin
+          - id: macos-15-arm64
+            os: macos-15
+            target: aarch64-apple-darwin
+          - id: macos-26-x86_64
+            os: macos-26-intel
+            target: x86_64-apple-darwin
+          - id: macos-26-arm64
+            os: macos-26
+            target: aarch64-apple-darwin
+          - id: windows-2022-x86_64
+            os: windows-2022
+            target: x86_64-pc-windows-msvc
+          - id: windows-2025-x86_64
+            os: windows-2025
+            target: x86_64-pc-windows-msvc
+          - id: windows-11-arm64
+            os: windows-11-arm
+            target: aarch64-pc-windows-msvc
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ needs.release-target.outputs.tag }}
+          persist-credentials: false
+          submodules: true
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Verify native Rust target
+        shell: bash
+        env:
+          EXPECTED_TARGET: ${{ matrix.target }}
+        run: |
+          set -euo pipefail
+          actual_target="$(rustc -vV | sed -n 's/^host: //p')"
+          [[ "$actual_target" == "$EXPECTED_TARGET" ]]
+
+      - name: Build installable module
+        run: cargo build --locked --release --manifest-path crates/tinymemory-module/Cargo.toml
+
+      - name: Assemble Unix module package
+        if: ${{ runner.os != 'Windows' }}
+        id: unix_package
+        shell: bash
+        env:
+          BUNDLE_ID: ${{ matrix.id }}
+          VERSION: ${{ needs.release-target.outputs.next_version }}
+        run: |
+          set -euo pipefail
+
+          library_name="tinymemory_module"
+          case "$RUNNER_OS" in
+            Linux) module="crates/tinymemory-module/target/release/lib${library_name}.so" ;;
+            macOS) module="crates/tinymemory-module/target/release/lib${library_name}.dylib" ;;
+            *) echo "unsupported Unix runner: ${RUNNER_OS}" >&2; exit 1 ;;
+          esac
+          package_name="tinymemory-module-${VERSION}-${BUNDLE_ID}"
+          package_root="dist/${package_name}"
+          mkdir -p "$package_root"
+          install -m 755 "$module" "$package_root/"
+          install -m 644 LICENSE README.md docs/specs/tinybus-module.md "$package_root/"
+          module_name="$(basename "$module")"
+          module_hash="$(sha256sum "$package_root/$module_name" | awk '{print $1}')"
+          printf '"%s" = "%s"\n' "$module_name" "$module_hash" \
+            > "$package_root/modules.toml"
+          tar -C "$package_root" -czf "dist/${package_name}.tar.gz" .
+          echo "archive=dist/${package_name}.tar.gz" >> "$GITHUB_OUTPUT"
+
+      - name: Assemble Windows module package
+        if: ${{ runner.os == 'Windows' }}
+        id: windows_package
+        shell: pwsh
+        env:
+          BUNDLE_ID: ${{ matrix.id }}
+          VERSION: ${{ needs.release-target.outputs.next_version }}
+        run: |
+          $ErrorActionPreference = 'Stop'
+          $libraryName = 'tinymemory_module'
+          $module = "crates/tinymemory-module/target/release/$libraryName.dll"
+          $packageName = "tinymemory-module-$env:VERSION-$env:BUNDLE_ID"
+          $packageRoot = "dist/$packageName"
+          New-Item -ItemType Directory -Force $packageRoot | Out-Null
+          Copy-Item -LiteralPath $module, 'LICENSE', 'README.md' -Destination $packageRoot
+          $hash = (Get-FileHash -LiteralPath $module -Algorithm SHA256).Hash.ToLowerInvariant()
+          $moduleName = Split-Path -Leaf $module
+          "`"$moduleName`" = `"$hash`"`n" |
+            Set-Content -Path "$packageRoot/modules.toml" -Encoding utf8NoBOM
+          Compress-Archive -Path "$packageRoot/*" -DestinationPath "dist/$packageName.zip"
+          "archive=dist/$packageName.zip" >> $env:GITHUB_OUTPUT
+
+      - name: Upload Unix module package
+        if: ${{ runner.os != 'Windows' }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: tinymemory-module-${{ matrix.id }}
+          path: ${{ steps.unix_package.outputs.archive }}
+          if-no-files-found: error
+
+      - name: Upload Windows module package
+        if: ${{ runner.os == 'Windows' }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: tinymemory-module-${{ matrix.id }}
+          path: ${{ steps.windows_package.outputs.archive }}
+          if-no-files-found: error
+
+  github-release:
+    name: Create GitHub release
+    needs:
+      - release-target
+      - native-bundles
+    # Same transitive-skip rule as above.
+    if: >-
+      ${{ always()
+      && needs.release-target.result == 'success'
+      && needs.native-bundles.result == 'success' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ needs.release-target.outputs.tag }}
+          persist-credentials: false
+          submodules: true
+
+      - name: Download workflow artifacts
+        uses: actions/download-artifact@v8
+        with:
+          pattern: tinymemory-module-*
+          path: release-assets
+          merge-multiple: true
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - name: Create release checksum manifest with TinyBus
+        shell: bash
+        run: |
+          set -euo pipefail
+          mapfile -t assets < <(
+            find release-assets -type f \
+              \( -name '*.tar.gz' -o -name '*.zip' \) \
+              | sort
+          )
+          if [[ ${#assets[@]} -ne 11 ]]; then
+            printf 'expected 11 module archives, found %s:\n' "${#assets[@]}" >&2
+            find release-assets -type f -print >&2 || true
+            exit 1
+          fi
+          checksum_args=()
+          for asset in "${assets[@]}"; do checksum_args+=(--path "$asset"); done
+          cargo run --manifest-path vendor/tinybus/Cargo.toml --locked \
+            --package tinybus --all-features --bin tinybus -- \
+            modules checksum "${checksum_args[@]}" --output release-assets/checksum.toml
+
+      - name: Create release and upload assets
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RELEASE_TAG: ${{ needs.release-target.outputs.tag }}
+          REPOSITORY: ${{ github.repository }}
+        run: |
+          set -euo pipefail
+          mapfile -t release_files < <(find release-assets -type f | sort)
+          gh release create "$RELEASE_TAG" "${release_files[@]}" \
+            --repo "$REPOSITORY" \
+            --verify-tag \
+            --title "$RELEASE_TAG" \
+            --generate-notes
+
+      - name: Verify the published module through TinyBus
+        shell: bash
         env:
-          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          RELEASE_TAG: ${{ needs.release-target.outputs.tag }}
+          REPOSITORY: ${{ github.repository }}
+          VERSION: ${{ needs.release-target.outputs.next_version }}
+        run: |
+          set -euo pipefail
+          archive="tinymemory-module-${VERSION}-ubuntu-24.04-x86_64.tar.gz"
+          release_url="https://github.com/${REPOSITORY}/releases/tag/${RELEASE_TAG}"
+          sha256="$(sed -n "s/^\"${archive}\" = \"\([0-9a-f]\{64\}\)\"$/\1/p" release-assets/checksum.toml)"
+          test -n "$sha256"
+          cargo run --manifest-path vendor/tinybus/Cargo.toml --locked \
+            --package tinybus --all-features --example github_module_host -- \
+            "$release_url" "$archive" "$sha256"

.gitignore

@@ -1,5 +1,8 @@
-# Build output
-/target/
+# Build output. NOT anchored with a leading slash: `crates/tinymemory-module` is
+# its own workspace root (see the `exclude` note in Cargo.toml) and so has its
+# own `target/`, which an anchored `/target/` does not match — 3064 build files
+# and a 33 MB cdylib were committed before this was widened.
+target/
 **/*.rs.bk
 *.pdb