Develop

.github/workflows/ci.yml

@@ -21,6 +21,7 @@ jobs:
 
   backend:
     strategy:
+      fail-fast: false
       matrix:
         include:
           - os: ubuntu-latest
@@ -29,7 +30,7 @@ jobs:
           - os: windows-latest
             python: '3.12'
             managed-python: ./.venv/Scripts/python.exe
-          - os: macos-14
+          - os: macos-15
             python: '3.12'
             managed-python: ./.venv/bin/python
     runs-on: ${{ matrix.os }}
@@ -40,8 +41,10 @@ jobs:
         with: { python-version: '${{ matrix.python }}' }
       - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
         with: { version: '0.11.26' }
-      - run: python -m modiff.install --accelerator cpu --backend-only --non-interactive --json
+      - run: uv run --no-project --no-sync --python 3.12 -m modiff.dev plan --accelerator cpu --backend-only --json
+      - run: uv run --no-project --no-sync --python 3.12 -m modiff.dev setup --accelerator cpu --backend-only --non-interactive --json
       - run: uv pip install --python ${{ matrix.managed-python }} -r requirements/test.txt
       - run: uv pip check --python ${{ matrix.managed-python }}
-      - run: ${{ matrix.managed-python }} -m modiff.preflight --json --check-port 8088 --fail-on-error
+      - run: uv run --no-project --no-sync --python 3.12 -m modiff.dev check --json --check-port 8088 --fail-on-error
+      - run: ${{ matrix.managed-python }} scripts/smoke_service_package.py
       - run: ${{ matrix.managed-python }} -m pytest -q

.github/workflows/qualify-optional-runtime-macos.yml

@@ -0,0 +1,100 @@
+name: Qualify optional runtime on macOS
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: optional-runtime-macos-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  qualify-macos-arm64:
+    runs-on: macos-15
+    timeout-minutes: 35
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      - name: Verify hosted target
+        run: |
+          test "$(uname -m)" = "arm64"
+          sw_vers
+
+      - name: Prepare prospective clean base
+        run: |
+          cat > "$RUNNER_TEMP/prospective-base.patch" <<'PATCH'
+          diff --git a/pyproject.toml b/pyproject.toml
+          --- a/pyproject.toml
+          +++ b/pyproject.toml
+          @@ -12,16 +12,14 @@ dependencies = [
+             "aiohttp-cors>=0.7.0",
+             "kornia>=0.8.1",
+             "nanoid>=2.0.0",
+             # Keep unqualified targets base-delivered until their executable qualifier
+             # passes; Linux/Windows x86-64 receive the exact optional overlay on use.
+          -  "peft>=0.17.0; sys_platform == 'darwin' or platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'ARM64'",
+             "protobuf>=6.31.1",
+             "scipy>=1.15.2",
+             "sentencepiece>=0.2.0",
+             "torch>=2.6.0",
+             "torchsde>=0.2.6",
+             "torchvision>=0.21.0",
+          -  "transformers>=4.49.0; sys_platform == 'darwin' or platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'ARM64'",
+             "diffusers @ git+https://github.com/huggingface/diffusers.git@fbf49e7f35857f76bc57b177e26f12b03687c668",
+             "ftfy>=6.3.1",
+             "huggingface-hub>=1.31.0,<2.0",
+          PATCH
+          git apply --check "$RUNNER_TEMP/prospective-base.patch"
+          git apply "$RUNNER_TEMP/prospective-base.patch"
+          test "$(git diff --name-only)" = "pyproject.toml"
+          git diff --check
+          git diff -- pyproject.toml > "$RUNNER_TEMP/prospective-base.diff"
+          git config user.name "MoDiff qualification"
+          git config user.email "qualification@modiff.invalid"
+          git add -- pyproject.toml
+          git commit -m "chore: prepare prospective macOS qualification base"
+          test -z "$(git status --porcelain)"
+
+      - name: Install isolated CPU base
+        run: ./install.sh --accelerator cpu --backend-only --non-interactive
+
+      - name: Require clean-base qualification readiness
+        run: |
+          ./scripts/with-runtime-env.sh ./.venv/bin/python \
+            scripts/qualify_optional_runtime.py --preflight-only \
+            --profile-id huggingface-transformers-main-96fe6dce-peft-0.20.0 \
+            | tee "$RUNNER_TEMP/preflight.json"
+          ./.venv/bin/python -c 'import json, os, pathlib; value=json.loads(pathlib.Path(os.environ["RUNNER_TEMP"], "preflight.json").read_text()); assert value["status"] == "ready"; assert value["profileId"] == "huggingface-transformers-main-96fe6dce-peft-0.20.0"; assert value["platform"] == "macos"; assert value["machine"] == "arm64"; assert value["sourceRevisionReady"] is True; assert value["cleanBase"] is True; assert value["stagedPackagesPresent"] == []'
+
+      - name: Run explicit-consent qualification
+        run: |
+          ./scripts/with-runtime-env.sh ./.venv/bin/python \
+            scripts/qualify_optional_runtime.py --consent \
+            --profile-id huggingface-transformers-main-96fe6dce-peft-0.20.0 \
+            --evidence "$RUNNER_TEMP/optional-runtime-macos-arm64.json"
+          ./.venv/bin/python -c 'import json, os, pathlib; value=json.loads(pathlib.Path(os.environ["RUNNER_TEMP"], "optional-runtime-macos-arm64.json").read_text()); assert value["status"] == "passed"; assert value["profileId"] == "huggingface-transformers-main-96fe6dce-peft-0.20.0"; assert value["platform"] == "macos"; assert value["machine"] == "arm64"; assert value["sourceRevisionReady"] is True; assert value["cleanBase"] is True; assert value["managedStateRetained"] is False; assert value["sourceFlagsChanged"] is False'
+
+      - name: Record bounded target context
+        if: ${{ always() }}
+        run: |
+          {
+            git rev-parse HEAD
+            uname -m
+            sw_vers
+            test ! -x ./.venv/bin/python || ./.venv/bin/python --version
+          } > "$RUNNER_TEMP/target-context.txt"
+
+      - name: Upload review evidence
+        if: ${{ always() }}
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: optional-runtime-macos-arm64-${{ github.run_id }}
+          path: |
+            ${{ runner.temp }}/optional-runtime-macos-arm64.json
+            ${{ runner.temp }}/preflight.json
+            ${{ runner.temp }}/prospective-base.diff
+            ${{ runner.temp }}/target-context.txt
+          if-no-files-found: warn
+          retention-days: 14

.gitignore

@@ -9,8 +9,31 @@ config.ini
 *.local
 
 artifacts/
+# Local review notes, evidence, recordings, and handoff checkpoints.
+/review/
+/reviews/
+/review-assets/
+/checkpoints/
+/maintenance/
+/output/
+/playwright-report/
+/playwright-report-*/
+/blob-report/
+/test-results/
+/test-results-*/
+review-pending/
+review-approved/
 .pytest_cache/
 .ruff_cache/
+.mypy_cache/
+.hypothesis/
+.tox/
+.nox/
+.coverage
+.coverage.*
+coverage.xml
+htmlcov/
+build/
 *.egg-info/
 
 # Installer-managed downloads, toolchains, journals, and diagnostics.
@@ -23,6 +46,23 @@ custom/*
 
 data/*
 !data/model-artifact-catalog.json
+!data/image-prototyping-readiness.v1.json
+!data/huggingface-cluster-promotion-receipts.v1.json
+!data/huggingface-cluster-promotion-receipts.v2.json
+!data/huggingface-cluster-catalog-gates.v1.json
+!data/legacy-cluster-semantic-equivalence-receipts.v1.json
+!data/legacy-cluster-archived-definitions.v1.json
+!data/legacy-cluster-compiler-mappings.v1.json
+!data/legacy-cluster-studio-spec-evidence.v1.json
+!data/legacy-cluster-studio-spec-partial-reviews.v1.json
+!data/modular-block-contracts.json
+!data/modular-conditional-contracts.json
+!data/modular-workflow-contracts.json
+!data/anima-artifact-review.json
+!data/flux2-klein-base-4b-artifact-review.json
+!data/minimax-music3-artifact-review.json
+!data/qualification-graphs/
+!data/qualification-graphs/**
 !data/graphs/
 !data/workflow-library-manifest.json
 data/graphs/*

AGENTS.md

@@ -1,22 +1,32 @@
 # MoDiff Backend Agent Instructions
 
+Before node/Block/hierarchy, persistence/execution, qualification or cross-machine
+integration work, read [Cluster engineering lessons and required procedure](docs/cluster-engineering-lessons.md)
+completely. Treat its reproduction, preservation and proof-level checks as required,
+not optional background. Keep the mirrored guide in both repositories aligned.
+
 These rules apply to AI-assisted work in this repository. `CONTRIBUTING.md` is the complete contributor guide; read it together with `SECURITY.md` and the relevant document under `docs/` before editing.
 
 ## Scope And Runtime Boundary
 
 - Keep each change focused on a diagnosed problem. Trace the existing call path and tests before editing, and remove incidental generated files from the diff.
-- Hugging Face Diffusers and Modular Diffusers are MoDiff's only supported model-execution layer. Do not add an alternate graph executor, hosted inference provider, independent Transformers application, or another model driver.
-- Supporting libraries used by Diffusers and ordinary deterministic media processing are not alternate drivers. They must remain narrowly scoped, documented, and covered by tests.
+- MoDiff may execute models through an official library maintained and published by Hugging Face. A Hub repository, organization name, or compatible API is not enough: verify the library's upstream ownership and package provenance, then declare and review the exact integration in MoDiff's executable dependency contract.
+- All supported libraries run locally behind MoDiff's existing node graph, resource management, Auto/Expert, file, and security boundaries. Do not add an alternate graph executor, hosted inference provider, browser-side runtime, or library-owned workflow representation.
+- Keep nodes and client contracts task- or modality-generic. Library- and model-specific loading, parameter aliases, and output normalization belong in small backend adapters selected from a declared execution specification, not new model-named nodes or frontend branches.
+- Transformers is an optional runtime, not a default application dependency. Do not install it during startup, registry discovery, template browsing/opening, or Auto planning. A workflow that requires it must declare a reviewed optional runtime profile, present an explicit install/consent action, and verify the installed version before becoming runnable.
+- Ordinary deterministic media processing is allowed when narrowly scoped, documented, and covered by tests.
 - Never enable arbitrary remote Python code, mutable model revisions, or custom model execution implicitly. Trust-sensitive behavior requires an explicit operator choice and an immutable revision.
 - Prefer existing module, graph, configuration, error, and test patterns. Do not create a parallel workflow representation or model-loading path.
 
-## Diffusers And Modular Diffusers
+## Hugging Face Model Libraries
 
 - Keep the reviewed Diffusers revision pinned in the executable installation contract and update its compatibility test when changing it.
+- Treat support for each additional official Hugging Face library as an explicit integration: document its purpose and provenance, constrain its compatible version, keep heavyweight imports lazy, and add no-download compatibility and boundary tests.
+- Official-library eligibility is not blanket trust for Hub artifacts or repository code. Prefer `safetensors`; pin curated models and adapters immutably; and require a separate explicit operator decision for any reviewed remote-code path.
 - Use Diffusers loaders, pipelines, components, schedulers, adapters, and offload hooks instead of reimplementing upstream behavior.
 - Modular blocks should declare inputs, outputs, and dependencies clearly, avoid hidden cross-block state, and remain composable through `init_pipeline`.
 - Keep model-specific differences explicit and small. Put reusable behavior in the existing shared Diffusers modules rather than copying it into another pipeline.
-- Prefer `safetensors`; document and test any unavoidable unsafe deserialization or remote-code boundary.
+- For non-Diffusers libraries, put reusable behavior in the corresponding generic task module and preserve the same graph, resource, progress, cancellation, and output contracts.
 
 ## Security And Data
 
@@ -26,6 +36,12 @@ These rules apply to AI-assisted work in this repository. `CONTRIBUTING.md` is t
 - Never commit `config.ini`, tokens, local paths, generated outputs, model caches, qualification workspaces, virtual environments, logs, or template media.
 - Public template media belongs in the configured public Hugging Face Dataset repository. Keep only its versioned source descriptor, hashes, and documentation in Git.
 
+## Custom extensions
+
+- Read [Custom node development](docs/custom-nodes.md) before changing extension discovery, staging, enable, reload, or execution. Keep approvals outside source packages and bind them to inspected source and declared dependency versions.
+- Executable custom Python uses explicit Add/Load/Reload and the existing executor. Bind source/dependency hashes internally; discovery, preview and graph import are not consent. Contract-only Blocks remain non-executable.
+- A custom resource declaration is operator-reviewed code metadata, not catalog or hardware qualification. Do not execute custom suppliers during Auto inspection or assume their Python references are safe for early model eviction.
+
 ## Quality And Evidence
 
 - Add a regression test that fails for the original defect and covers related instances of the same pattern.

CONTRIBUTING.md

@@ -4,8 +4,14 @@ MoDiff is an experimental local backend with a separately maintained frontend bu
 
 Before starting, read [SECURITY.md](SECURITY.md) and the relevant guide in [docs/README.md](docs/README.md).
 
+Before changing nodes, Blocks, graph persistence/execution, model qualification or
+integrating another machine's work, read and follow
+[Cluster engineering lessons and required procedure](docs/cluster-engineering-lessons.md).
+
 ## Development setup
 
+For script-free `uv`/`npm` setup, use [Developer setup](docs/developer-setup.md).
+
 Use Python 3.12 and create the same managed CPU profile used by baseline CI:
 
 ```bash
@@ -22,7 +28,7 @@ uv pip install --python .venv/Scripts/python.exe -r requirements/test.txt
 .\.venv\Scripts\python.exe -m modiff.preflight --json --check-port 8088 --fail-on-error
 ```
 
-Choose the qualified accelerator profile relevant to a hardware-specific change and report that validation separately. Do not use `uv sync` or `uv run`: the project is intentionally `uv`-unmanaged because the installer, not the generic resolver, owns the executable Torch profile.
+Choose the qualified accelerator profile relevant to a hardware-specific change and report that validation separately. Do not use `uv sync` or ordinary `uv run` (the documented `uv run --no-project --no-sync ... -m modiff.dev` bootstrap is the explicit exception): the project is intentionally `uv`-unmanaged because the installer, not the generic resolver, owns the executable Torch profile.
 
 Do not commit `config.ini`, `.env` files, model caches, generated outputs, local logs, virtual environments, or test caches.
 
@@ -34,6 +40,10 @@ Do not commit `config.ini`, `.env` files, model caches, generated outputs, local
 - Treat file access, custom-module installation, remote code, token handling, and mutating routes as security-sensitive changes.
 - Avoid importing the full model registry from lightweight diagnostics such as preflight.
 
+### Custom node development
+
+For local/Git Python nodes and pinned Hub Modular blocks, follow [Custom node development](docs/custom-nodes.md). Stage and inspect without imports, explicitly enable the exact code hash, then review/reload after edits. Do not add an unconditional startup import or install dependencies from discovery. Test stale code, relative helper isolation, approval rejection, import diagnostics, and cache ownership through ordinary graph dispatch.
+
 ### Adding a node module
 
 Built-in node packages live under `modules/<Name>/` and normally contain:
@@ -65,6 +75,10 @@ uv pip check --python .venv/bin/python
 
 Use the corresponding accelerator instead of `cpu` when the change affects CUDA, ROCm, or MPS. Update the compatibility manifest and public installation guidance only when the evidence supports the claim. MoDiff deliberately has no `uv.lock`; do not generate one or describe the top-level requirements files as a cross-platform lock.
 
+MoDiff may integrate model libraries officially maintained and published by Hugging Face, but each library remains a separately reviewed execution dependency. Verify its upstream ownership, package provenance, license, supported version, loading behavior, and remote-code boundary. Hub hosting alone does not establish that a library or model is maintained by Hugging Face. Keep execution in the existing backend graph and expose task-generic contracts to the client.
+
+Transformers-specific execution is opt-in. Do not add Transformers to the base application environment or install an optional runtime merely because a template is viewed, nodes are discovered, or Auto compatibility is planned. A requiring workflow must identify its versioned optional runtime, show an explicit install/consent action, perform installation outside graph execution, and re-run package and compatibility checks before the workflow can run.
+
 ## Validation
 
 The baseline backend checks are:
@@ -78,6 +92,31 @@ uv pip check --python .venv/bin/python
 
 The wrapper applies the installed accelerator profile's process environment before Python imports Torch. On Windows, run `.venv/Scripts/python.exe` directly in the equivalent commands.
 
+To exercise tests requiring an **already installed and activated** optional
+runtime, use the verified entry point instead of adding a sealed site-packages
+directory to `PYTHONPATH` manually:
+
+```bash
+./scripts/with-runtime-env.sh .venv/bin/python scripts/test_reviewed_optional_runtime.py -q tests
+```
+
+It validates the active artifact-locked runtime through the normal startup
+boundary, performs no installation, and disables bytecode writes in this process
+and child generators. Generated `__pycache__` files inside a sealed overlay are
+integrity drift, not files to whitelist. Keep base-gate and optional-runtime
+results separate; skipped model-library tests are not execution coverage.
+
+Pytest redirects the default extension store to a temporary directory before
+collection so registry imports cannot execute or change the operator's installed
+custom sources. Extension tests use explicit temporary roots. Subprocess tests
+must also isolate extension discovery; they do not inherit Python monkeypatches.
+
+The checked-in image readiness inventory uses a fixed Linux/x86_64 reference
+target; `scripts/generate_image_prototyping_readiness.py --check` must reproduce
+it on every host. This static inventory is not local runtime readiness. Live
+execution profiles continue to resolve the current OS/architecture and installed
+runtime through the normal readiness boundary.
+
 On a host with Git Bash or a POSIX shell:
 
 ```bash
@@ -104,6 +143,18 @@ See [README.md](README.md#updating-the-bundled-client) for exact-mirror examples
 
 Update public documentation when commands, extras, routes, storage behavior, trust boundaries, or compatibility guarantees change. Keep claims proportional to the evidence actually run.
 
+Put lasting guidance in the relevant API, runtime, module, or troubleshooting
+guide and link it from `docs/README.md`. Keep dated execution trackers, private
+task IDs, raw logs, screenshots, recordings, and handoff notes outside public
+documentation. Local `reviews/`, `checkpoints/`, and `maintenance/` directories
+are ignored; sanitized evidence can be attached to the issue or pull request.
+
+Before committing, inspect `git status --short`, `git diff --cached --stat`, and
+`git diff --cached --check`, then review the staged content. Ignore rules do not
+remove already tracked files. Keep intentional fixtures, executable catalogs,
+and the checked frontend bundle; do not confuse these required artifacts with
+local reports, cache contents, or generated model output.
+
 A useful change description includes:
 
 - The user-visible problem and intended behavior.