diff --git a/README.md b/README.md index 4669b05..a0d46ba 100644 --- a/README.md +++ b/README.md @@ -103,3 +103,19 @@ Integration tests are run by pytest tests/ This will perform all the tests in `tests/` and its subfolders. + +Several tests exercise libneo's EQDSK, Boozer, MGRID and `efit_to_boozer` +readers against real experimental and design equilibria stored in the shared +`gitlab.tugraz.at/plasma/data` repository. To fetch the required subtrees into +`$DATA`, set a read-only GitLab token and run + + export GITLAB_ACCESS_TOKEN= + export DATA=$PWD/.testdata + scripts/fetch_data.sh AUG/EQDSK AUG/BOOZER/30835 DEMO/EQDSK MASTU/EQDSK \ + LHD/VMEC/makegrid_alternative TESTS/libneo/eqdsk + pytest tests/ + +Without a token (or a partial fetch), `scripts/fetch_data.sh` skips gracefully +and the data-dependent tests skip instead of failing. GitHub Actions runs this +automatically in `.github/workflows/tests.yml`, fetching DATA when +`GITLAB_ACCESS_TOKEN` is available and otherwise reporting passes and skips. diff --git a/scripts/fetch_data.sh b/scripts/fetch_data.sh new file mode 100755 index 0000000..640877e --- /dev/null +++ b/scripts/fetch_data.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Fetch selected subtrees of the shared ITPcp plasma data repository +# (gitlab.tugraz.at/plasma/data) into $DATA so the integration tests in +# tests/ can run against real experimental and design equilibria. +# +# Authentication is via a read-only GitLab token in GITLAB_ACCESS_TOKEN. +# Without a token the script exits 0 and tests relying on $DATA skip. +# +# Usage: scripts/fetch_data.sh [ ...] +# Example: scripts/fetch_data.sh AUG/EQDSK AUG/BOOZER/30835 + +if [[ -z "${GITLAB_ACCESS_TOKEN:-}" ]]; then + echo "GITLAB_ACCESS_TOKEN is not set; skipping DATA fetch" + exit 0 +fi + +if [[ $# -eq 0 ]]; then + echo "usage: $0 [ ...]" >&2 + exit 2 +fi + +repo_url="${GITLAB_DATA_REPO_URL:-https://oauth2:${GITLAB_ACCESS_TOKEN}@gitlab.tugraz.at/plasma/data.git}" +data_root="${DATA:-$(pwd)/.testdata}" +branch="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME:-main}}" + +# Prefer a sparse, blob-free clone so we only pull the requested subtrees +# (LFS blobs are filtered further below). Full clones of plasma/data are slow. +tmpdir="$(mktemp -d)" +cleanup() { + rm -rf "$tmpdir" +} +trap cleanup EXIT + +echo "Fetching DATA subtrees ($*) on branch ${branch}" +git clone --filter=blob:none --sparse --no-checkout "$repo_url" "$tmpdir/data" +cd "$tmpdir/data" + +if git ls-remote --exit-code --heads origin "$branch" >/dev/null 2>&1; then + git checkout "$branch" 2>/dev/null || true +else + echo "Branch ${branch} not found in data repo; falling back to main" + git checkout main 2>/dev/null || true +fi + +git sparse-checkout set "$@" + +if [[ -f .gitattributes ]] && grep -q 'filter=lfs' .gitattributes; then + git lfs install --local >/dev/null 2>&1 || true + git config lfs.fetchinclude "$(IFS=,; echo "$*")" + git config lfs.fetchexclude "" + git lfs pull 2>/dev/null || true + git lfs checkout "$@" 2>/dev/null || true +fi + +# Only copy the requested subtrees into $DATA. +mkdir -p "$data_root" +for subdir in "$@"; do + if [[ ! -d "$subdir" ]]; then + echo "Subtree ${subdir} not found in data repo; skipping" + continue + fi + mkdir -p "$data_root/$(dirname "$subdir")" + cp -a "$subdir" "$data_root/$subdir" +done +echo "DATA subtrees synchronized to $data_root" diff --git a/tests/NEO-RT/benchmark_with_NEO_2/test_util.py b/tests/NEO-RT/benchmark_with_NEO_2/test_util.py index 6763dfe..0426d7a 100644 --- a/tests/NEO-RT/benchmark_with_NEO_2/test_util.py +++ b/tests/NEO-RT/benchmark_with_NEO_2/test_util.py @@ -7,5 +7,3 @@ def test_replace_template(): } text = fill_template(template, vars) assert(text == 'bla 1 blu bli') - -test_replace_template() diff --git a/tests/conftest.py b/tests/conftest.py index 035d0df..88f1524 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,60 @@ -import pytest - import os from pathlib import Path +import pytest + + +def pytest_addoption(parser): + parser.addoption( + "--regenerate-golden", + action="store_true", + default=False, + help="Regenerate (overwrite) golden records instead of only verifying them", + ) + + @pytest.fixture def code_path(): - return Path(os.environ["CODE"]) + # Default to the workspace/repository root when $CODE is unset (e.g. CI). + root = Path(os.environ.get("CODE", Path(__file__).resolve().parents[1])) + return Path(root) + @pytest.fixture def data_path(): - return Path(os.environ["DATA"]) + """Return the shared data root, skipping when no DATA is available. + + Resolves $DATA, else a local .testdata directory next to the repository + (populated by scripts/fetch_data.sh), else skips. + """ + data = os.environ.get("DATA") + if data: + return Path(data) + local = Path(__file__).resolve().parents[1] / ".testdata" + if local.exists(): + return local + pytest.skip("DATA not available: set $DATA or run scripts/fetch_data.sh") + + +@pytest.fixture +def require_data(data_path): + """Return a callable asserting the given relative paths exist in DATA. + + Skips the requesting test when any path is missing so a partial fetch + only skips the affected tests instead of erroring in fixture setup. + """ + + def _require_data(*relpaths): + missing = [] + for rel in relpaths: + if not (data_path / rel).exists(): + missing.append(rel) + if missing: + pytest.skip( + "DATA not available: missing " + + ", ".join(missing) + + "; run scripts/fetch_data.sh" + ) + return [data_path / rel for rel in relpaths] + + return _require_data diff --git a/tests/libneo/python/test_boozer.py b/tests/libneo/python/test_boozer.py index 4bf6bbe..7cdbb09 100644 --- a/tests/libneo/python/test_boozer.py +++ b/tests/libneo/python/test_boozer.py @@ -10,14 +10,14 @@ @pytest.fixture -def boozer(data_path): - trial_boozer_file = data_path / "AUG/BOOZER/30835/out_neo-2_rmp_90-n0" +def boozer(require_data): + (trial_boozer_file,) = require_data("AUG/BOOZER/30835/out_neo-2_rmp_90-n0") return BoozerFile(str(trial_boozer_file)) @pytest.fixture -def eqdsk(data_path): - trial_eqdsk_file = data_path / "AUG/EQDSK/g30835.3200_ed6" +def eqdsk(require_data): + (trial_eqdsk_file,) = require_data("AUG/EQDSK/g30835.3200_ed6") return read_eqdsk(str(trial_eqdsk_file)) diff --git a/tests/libneo/python/test_efit_to_boozer.py b/tests/libneo/python/test_efit_to_boozer.py index bfd2c05..0f229f3 100644 --- a/tests/libneo/python/test_efit_to_boozer.py +++ b/tests/libneo/python/test_efit_to_boozer.py @@ -7,7 +7,7 @@ from numpy.testing import assert_allclose import matplotlib.pyplot as plt -import _efit_to_boozer as efit_to_boozer +efit_to_boozer = pytest.importorskip("_efit_to_boozer") from libneo import read_eqdsk, FluxConverter efit_to_boozer_input = """3600 nstep - number of integration steps @@ -34,12 +34,20 @@ """ @pytest.fixture -def test_files(code_path, data_path): +def test_files(code_path, require_data): + data = dict( + zip( + ["DEMO CHEASE", "AUG", "MASTU"], + require_data( + "DEMO/EQDSK/Equilibrium_DEMO2019_CHEASE/MOD_Qprof_Test/EQDSK_DEMO2019_q1_COCOS_02.OUT", + "AUG/EQDSK/g30835.3200_ed6", + "MASTU/EQDSK/MAST_47051_450ms.geqdsk", + ), + ) + ) return { "local": code_path / "libneo/test/resources/input_efit_file.dat", - "DEMO CHEASE": data_path / "DEMO/EQDSK/Equilibrium_DEMO2019_CHEASE/MOD_Qprof_Test/EQDSK_DEMO2019_q1_COCOS_02.OUT", - "AUG": data_path / "AUG/EQDSK/g30835.3200_ed6", - "MASTU": data_path / "MASTU/EQDSK/MAST_47051_450ms.geqdsk", + **data, # TODO: "DEMO PROCESS": data_path / "DEMO/EQDSK/Equil_2021_PMI_QH_mode_betap_1d04_li_1d02_Ip_18d27MA_SOF.eqdsk", # TODO: "DEMO standardized": data_path / "DEMO/EQDSK/Equil_2021_PMI_QH_mode_betap_1d04_li_1d02_Ip_18d27MA_SOF_std.eqdsk", } diff --git a/tests/libneo/python/test_eqdsk.py b/tests/libneo/python/test_eqdsk.py index 8272d83..0fb9864 100644 --- a/tests/libneo/python/test_eqdsk.py +++ b/tests/libneo/python/test_eqdsk.py @@ -10,15 +10,22 @@ from libneo import eqdsk +# Reference EQDSK files stored in the shared DATA repository. +DATA_EQDSK_FILES = { + "PROCESS": "DEMO/EQDSK/Equil_2021_PMI_QH_mode_betap_1d04_li_1d02_Ip_18d27MA_SOF.eqdsk", + "standardized": "DEMO/EQDSK/Equil_2021_PMI_QH_mode_betap_1d04_li_1d02_Ip_18d27MA_SOF_std.eqdsk", + "AUG": "AUG/EQDSK/g30835.3200_ed6", + # TODO: "MASTU": "MASTU/EQDSK/MAST_47051_450ms.geqdsk", + # TODO: "CHEASE": "DEMO/teams/Equilibrium_DEMO2019_CHEASE/MOD_Qprof_Test/EQDSK_DEMO2019_q1_COCOS_02.OUT", +} + + @pytest.fixture -def test_files(code_path, data_path): +def test_files(code_path, require_data): + data = dict(zip(DATA_EQDSK_FILES, require_data(*DATA_EQDSK_FILES.values()))) return { "local": code_path / "libneo/test/resources/input_efit_file.dat", - "PROCESS": data_path / "DEMO/EQDSK/Equil_2021_PMI_QH_mode_betap_1d04_li_1d02_Ip_18d27MA_SOF.eqdsk", - "standardized": data_path / "DEMO/EQDSK/Equil_2021_PMI_QH_mode_betap_1d04_li_1d02_Ip_18d27MA_SOF_std.eqdsk", - "AUG": data_path / "AUG/EQDSK/g30835.3200_ed6", - # TODO: "MASTU": data_path / "MASTU/EQDSK/MAST_47051_450ms.geqdsk", - # TODO: "CHEASE": data_path / "DEMO/teams/Equilibrium_DEMO2019_CHEASE/MOD_Qprof_Test/EQDSK_DEMO2019_q1_COCOS_02.OUT", + **data, } @@ -29,9 +36,13 @@ def test_eqdsk_read(test_files): @pytest.mark.slow -def test_eqdsk_golden_records(data_path, test_files): - golden_record_path = data_path / "TESTS/libneo/eqdsk" - store_golden_records(test_files.values(), golden_record_path) +def test_eqdsk_golden_records(request, test_files, require_data): + (golden_record_path,) = require_data("TESTS/libneo/eqdsk") + + # Golden records are compared read-only unless explicitly regenerated, + # so a CI run never silently writes into the shared DATA tree. + if request.config.getoption("--regenerate-golden"): + store_golden_records(test_files.values(), golden_record_path, force_overwrite=True) for test_file in test_files.values(): eqdsk_object = eqdsk.eqdsk_file(test_file) diff --git a/tests/libneo/python/test_mgrid.py b/tests/libneo/python/test_mgrid.py index 6407afb..6cb2793 100644 --- a/tests/libneo/python/test_mgrid.py +++ b/tests/libneo/python/test_mgrid.py @@ -5,8 +5,9 @@ from libneo.mgrid import MgridFile @pytest.fixture -def testfile_path(data_path): - return data_path / "LHD/VMEC/makegrid_alternative/mgrid_lhd_nfp10.nc" +def testfile_path(require_data): + (testfile,) = require_data("LHD/VMEC/makegrid_alternative/mgrid_lhd_nfp10.nc") + return testfile @pytest.fixture