tests: get DATA into tests (fetch with auth token, skip when unavailable)

README.md

@@ -103,3 +103,19 @@ Integration tests are run by
     pytest tests/
 
 This will perform all the tests in `tests/` and its subfolders.
+
+Several tests exercise libneo's EQDSK, Boozer, MGRID and `efit_to_boozer`
+readers against real experimental and design equilibria stored in the shared
+`gitlab.tugraz.at/plasma/data` repository. To fetch the required subtrees into
+`$DATA`, set a read-only GitLab token and run
+
+    export GITLAB_ACCESS_TOKEN=<read-only token>
+    export DATA=$PWD/.testdata
+    scripts/fetch_data.sh AUG/EQDSK AUG/BOOZER/30835 DEMO/EQDSK MASTU/EQDSK \
+        LHD/VMEC/makegrid_alternative TESTS/libneo/eqdsk
+    pytest tests/
+
+Without a token (or a partial fetch), `scripts/fetch_data.sh` skips gracefully
+and the data-dependent tests skip instead of failing. GitHub Actions runs this
+automatically in `.github/workflows/tests.yml`, fetching DATA when
+`GITLAB_ACCESS_TOKEN` is available and otherwise reporting passes and skips.

scripts/fetch_data.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Fetch selected subtrees of the shared ITPcp plasma data repository
+# (gitlab.tugraz.at/plasma/data) into $DATA so the integration tests in
+# tests/ can run against real experimental and design equilibria.
+#
+# Authentication is via a read-only GitLab token in GITLAB_ACCESS_TOKEN.
+# Without a token the script exits 0 and tests relying on $DATA skip.
+#
+# Usage: scripts/fetch_data.sh <subdir> [<subdir> ...]
+# Example: scripts/fetch_data.sh AUG/EQDSK AUG/BOOZER/30835
+
+if [[ -z "${GITLAB_ACCESS_TOKEN:-}" ]]; then
+    echo "GITLAB_ACCESS_TOKEN is not set; skipping DATA fetch"
+    exit 0
+fi
+
+if [[ $# -eq 0 ]]; then
+    echo "usage: $0 <subdir> [<subdir> ...]" >&2
+    exit 2
+fi
+
+repo_url="${GITLAB_DATA_REPO_URL:-https://oauth2:${GITLAB_ACCESS_TOKEN}@gitlab.tugraz.at/plasma/data.git}"
+data_root="${DATA:-$(pwd)/.testdata}"
+branch="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME:-main}}"
+
+# Prefer a sparse, blob-free clone so we only pull the requested subtrees
+# (LFS blobs are filtered further below). Full clones of plasma/data are slow.
+tmpdir="$(mktemp -d)"
+cleanup() {
+    rm -rf "$tmpdir"
+}
+trap cleanup EXIT
+
+echo "Fetching DATA subtrees ($*) on branch ${branch}"
+git clone --filter=blob:none --sparse --no-checkout "$repo_url" "$tmpdir/data"
+cd "$tmpdir/data"
+
+if git ls-remote --exit-code --heads origin "$branch" >/dev/null 2>&1; then
+    git checkout "$branch" 2>/dev/null || true
+else
+    echo "Branch ${branch} not found in data repo; falling back to main"
+    git checkout main 2>/dev/null || true
+fi
+
+git sparse-checkout set "$@"
+
+if [[ -f .gitattributes ]] && grep -q 'filter=lfs' .gitattributes; then
+    git lfs install --local >/dev/null 2>&1 || true
+    git config lfs.fetchinclude "$(IFS=,; echo "$*")"
+    git config lfs.fetchexclude ""
+    git lfs pull 2>/dev/null || true
+    git lfs checkout "$@" 2>/dev/null || true
+fi
+
+# Only copy the requested subtrees into $DATA.
+mkdir -p "$data_root"
+for subdir in "$@"; do
+    if [[ ! -d "$subdir" ]]; then
+        echo "Subtree ${subdir} not found in data repo; skipping"
+        continue
+    fi
+    mkdir -p "$data_root/$(dirname "$subdir")"
+    cp -a "$subdir" "$data_root/$subdir"
+done
+echo "DATA subtrees synchronized to $data_root"

tests/NEO-RT/benchmark_with_NEO_2/test_util.py

@@ -7,5 +7,3 @@ def test_replace_template():
     }
     text = fill_template(template, vars)
     assert(text == 'bla 1 blu bli')
-
-test_replace_template()

tests/conftest.py

@@ -1,12 +1,60 @@
-import pytest
-
 import os
 from pathlib import Path
 
+import pytest
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--regenerate-golden",
+        action="store_true",
+        default=False,
+        help="Regenerate (overwrite) golden records instead of only verifying them",
+    )
+
+
 @pytest.fixture
 def code_path():
-    return Path(os.environ["CODE"])
+    # Default to the workspace/repository root when $CODE is unset (e.g. CI).
+    root = Path(os.environ.get("CODE", Path(__file__).resolve().parents[1]))
+    return Path(root)
+
 
 @pytest.fixture
 def data_path():
-    return Path(os.environ["DATA"])
+    """Return the shared data root, skipping when no DATA is available.
+
+    Resolves $DATA, else a local .testdata directory next to the repository
+    (populated by scripts/fetch_data.sh), else skips.
+    """
+    data = os.environ.get("DATA")
+    if data:
+        return Path(data)
+    local = Path(__file__).resolve().parents[1] / ".testdata"
+    if local.exists():
+        return local
+    pytest.skip("DATA not available: set $DATA or run scripts/fetch_data.sh")
+
+
+@pytest.fixture
+def require_data(data_path):
+    """Return a callable asserting the given relative paths exist in DATA.
+
+    Skips the requesting test when any path is missing so a partial fetch
+    only skips the affected tests instead of erroring in fixture setup.
+    """
+
+    def _require_data(*relpaths):
+        missing = []
+        for rel in relpaths:
+            if not (data_path / rel).exists():
+                missing.append(rel)
+        if missing:
+            pytest.skip(
+                "DATA not available: missing "
+                + ", ".join(missing)
+                + "; run scripts/fetch_data.sh"
+            )
+        return [data_path / rel for rel in relpaths]
+
+    return _require_data

tests/libneo/python/test_boozer.py

@@ -10,14 +10,14 @@
 
 
 @pytest.fixture
-def boozer(data_path):
-    trial_boozer_file = data_path / "AUG/BOOZER/30835/out_neo-2_rmp_90-n0"
+def boozer(require_data):
+    (trial_boozer_file,) = require_data("AUG/BOOZER/30835/out_neo-2_rmp_90-n0")
     return BoozerFile(str(trial_boozer_file))
 
 
 @pytest.fixture
-def eqdsk(data_path):
-    trial_eqdsk_file = data_path / "AUG/EQDSK/g30835.3200_ed6"
+def eqdsk(require_data):
+    (trial_eqdsk_file,) = require_data("AUG/EQDSK/g30835.3200_ed6")
     return read_eqdsk(str(trial_eqdsk_file))
 
 

tests/libneo/python/test_efit_to_boozer.py

@@ -7,7 +7,7 @@
 from numpy.testing import assert_allclose
 import matplotlib.pyplot as plt
 
-import _efit_to_boozer as efit_to_boozer
+efit_to_boozer = pytest.importorskip("_efit_to_boozer")
 from libneo import read_eqdsk, FluxConverter
 
 efit_to_boozer_input = """3600      nstep    - number of integration steps
@@ -34,12 +34,20 @@
 """
 
 @pytest.fixture
-def test_files(code_path, data_path):
+def test_files(code_path, require_data):
+    data = dict(
+        zip(
+            ["DEMO CHEASE", "AUG", "MASTU"],
+            require_data(
+                "DEMO/EQDSK/Equilibrium_DEMO2019_CHEASE/MOD_Qprof_Test/EQDSK_DEMO2019_q1_COCOS_02.OUT",
+                "AUG/EQDSK/g30835.3200_ed6",
+                "MASTU/EQDSK/MAST_47051_450ms.geqdsk",
+            ),
+        )
+    )
     return {
         "local": code_path / "libneo/test/resources/input_efit_file.dat",
-        "DEMO CHEASE": data_path / "DEMO/EQDSK/Equilibrium_DEMO2019_CHEASE/MOD_Qprof_Test/EQDSK_DEMO2019_q1_COCOS_02.OUT",
-        "AUG": data_path / "AUG/EQDSK/g30835.3200_ed6",
-        "MASTU": data_path / "MASTU/EQDSK/MAST_47051_450ms.geqdsk",
+        **data,
         # TODO: "DEMO PROCESS": data_path / "DEMO/EQDSK/Equil_2021_PMI_QH_mode_betap_1d04_li_1d02_Ip_18d27MA_SOF.eqdsk",
         # TODO: "DEMO standardized": data_path / "DEMO/EQDSK/Equil_2021_PMI_QH_mode_betap_1d04_li_1d02_Ip_18d27MA_SOF_std.eqdsk",
     }

tests/libneo/python/test_eqdsk.py

@@ -10,15 +10,22 @@
 from libneo import eqdsk
 
 
+# Reference EQDSK files stored in the shared DATA repository.
+DATA_EQDSK_FILES = {
+    "PROCESS": "DEMO/EQDSK/Equil_2021_PMI_QH_mode_betap_1d04_li_1d02_Ip_18d27MA_SOF.eqdsk",
+    "standardized": "DEMO/EQDSK/Equil_2021_PMI_QH_mode_betap_1d04_li_1d02_Ip_18d27MA_SOF_std.eqdsk",
+    "AUG": "AUG/EQDSK/g30835.3200_ed6",
+    # TODO: "MASTU": "MASTU/EQDSK/MAST_47051_450ms.geqdsk",
+    # TODO: "CHEASE": "DEMO/teams/Equilibrium_DEMO2019_CHEASE/MOD_Qprof_Test/EQDSK_DEMO2019_q1_COCOS_02.OUT",
+}
+
+
 @pytest.fixture
-def test_files(code_path, data_path):
+def test_files(code_path, require_data):
+    data = dict(zip(DATA_EQDSK_FILES, require_data(*DATA_EQDSK_FILES.values())))
     return {
         "local": code_path / "libneo/test/resources/input_efit_file.dat",
-        "PROCESS": data_path / "DEMO/EQDSK/Equil_2021_PMI_QH_mode_betap_1d04_li_1d02_Ip_18d27MA_SOF.eqdsk",
-        "standardized": data_path / "DEMO/EQDSK/Equil_2021_PMI_QH_mode_betap_1d04_li_1d02_Ip_18d27MA_SOF_std.eqdsk",
-        "AUG": data_path / "AUG/EQDSK/g30835.3200_ed6",
-        # TODO: "MASTU": data_path / "MASTU/EQDSK/MAST_47051_450ms.geqdsk",
-        # TODO: "CHEASE": data_path / "DEMO/teams/Equilibrium_DEMO2019_CHEASE/MOD_Qprof_Test/EQDSK_DEMO2019_q1_COCOS_02.OUT",
+        **data,
     }
 
 
@@ -29,9 +36,13 @@ def test_eqdsk_read(test_files):
 
 
 @pytest.mark.slow
-def test_eqdsk_golden_records(data_path, test_files):
-    golden_record_path = data_path / "TESTS/libneo/eqdsk"
-    store_golden_records(test_files.values(), golden_record_path)
+def test_eqdsk_golden_records(request, test_files, require_data):
+    (golden_record_path,) = require_data("TESTS/libneo/eqdsk")
+
+    # Golden records are compared read-only unless explicitly regenerated,
+    # so a CI run never silently writes into the shared DATA tree.
+    if request.config.getoption("--regenerate-golden"):
+        store_golden_records(test_files.values(), golden_record_path, force_overwrite=True)
 
     for test_file in test_files.values():
         eqdsk_object = eqdsk.eqdsk_file(test_file)

tests/libneo/python/test_mgrid.py

@@ -5,8 +5,9 @@
 from libneo.mgrid import MgridFile
 
 @pytest.fixture
-def testfile_path(data_path):
-    return data_path / "LHD/VMEC/makegrid_alternative/mgrid_lhd_nfp10.nc"
+def testfile_path(require_data):
+    (testfile,) = require_data("LHD/VMEC/makegrid_alternative/mgrid_lhd_nfp10.nc")
+    return testfile
 
 
 @pytest.fixture