feat: faster integrated tests

.github/workflows/build_and_test.yml

@@ -49,6 +49,10 @@ on:
       GCP_BUCKET:
         required: false
         type: string
+      UPLOAD_TO_GCP:
+        required: false
+        type: boolean
+        default: false
       HOST_CONFIG:
         required: false
         type: string
@@ -59,6 +63,10 @@ on:
       RUNS_ON:
         required: true
         type: string
+      SCCACHE_HOST_DIR:
+        required: false
+        type: string
+        default: ''
       USE_SCCACHE:
         required: false
         type: boolean
@@ -127,7 +135,7 @@ jobs:
         fetch-depth: 1
 
     - id: 'auth'
-      if: ${{ inputs.GCP_BUCKET || inputs.USE_SCCACHE }}
+      if: ${{ (inputs.UPLOAD_TO_GCP && inputs.GCP_BUCKET) || (inputs.USE_SCCACHE && !startsWith(inputs.RUNS_ON, 'streak2')) }}
       uses: 'google-github-actions/auth@v2.1.10'
       with:
         credentials_json: '${{ secrets.GOOGLE_CLOUD_GCP }}'
@@ -146,7 +154,7 @@ jobs:
         echo "GNUPGHOME=${GNUPGHOME}" >> "${GITHUB_ENV}"
 
     - name: 'Set up Cloud SDK'
-      if: inputs.GCP_BUCKET
+      if: inputs.UPLOAD_TO_GCP && inputs.GCP_BUCKET
       uses: 'google-github-actions/setup-gcloud@v2.1.4'
       with:
         version: '>= 363.0.0'
@@ -158,6 +166,9 @@ jobs:
         docker_args=()
         script_args=()
 
+        # Keep the repository mount point consistent between Docker and script args.
+        GITHUB_WORKSPACE_MOUNT_POINT=/tmp/geos
+
         if [[ -n "${{ inputs.NPROC }}" ]]; then
           NPROC=${{ inputs.NPROC }}
           script_args+=(--nproc ${NPROC})
@@ -173,21 +184,25 @@ jobs:
         SHORT_COMMIT=${COMMIT:0:7}
         script_args+=(--install-dir-basename GEOS-${SHORT_COMMIT})
 
-        # All the data exchanged with the docker container is eventually meant to be sent to the cloud.
-        if [[ ! -z "${{ inputs.GCP_BUCKET }}" ]]; then
-          if [ "${{ inputs.BUILD_TYPE }}" = "build" ]; then
-            DATA_BASENAME=GEOS-and-TPL-${SHORT_COMMIT}.tar.gz
-          elif [ "${{ inputs.BUILD_TYPE }}" = "integrated_tests" ]; then
-            DATA_BASENAME=integratedTests-pr${{ github.event.number }}-${{ github.run_number }}-${SHORT_COMMIT}.tar.gz
-            script_args+=(--run-integrated-tests)
-          fi
+        if [[ "${{ inputs.UPLOAD_TO_GCP }}" == "true" && -z "${{ inputs.GCP_BUCKET }}" ]]; then
+          echo "::error::UPLOAD_TO_GCP=true requires GCP_BUCKET to be set."
+          exit 1
+        fi
+
+        DATA_BASENAME=
+        DATA_EXCHANGE_DIR=
+        if [[ "${{ inputs.BUILD_TYPE }}" = "integrated_tests" ]]; then
+          DATA_BASENAME=integratedTests-pr${{ github.event.number }}-${{ github.run_number }}-${SHORT_COMMIT}.tar.gz
+          script_args+=(--run-integrated-tests)
+        elif [[ "${{ inputs.UPLOAD_TO_GCP }}" == "true" && -n "${{ inputs.GCP_BUCKET }}" ]]; then
+          DATA_BASENAME=GEOS-and-TPL-${SHORT_COMMIT}.tar.gz
+        fi
 
+        if [[ -n "${DATA_BASENAME}" ]]; then
           script_args+=(--data-basename ${DATA_BASENAME})
 
           DATA_EXCHANGE_DIR=${GITHUB_WORKSPACE}/geos-exchange  # Exchange folder outside of the container
-          if [ ! -d "${DATA_EXCHANGE_DIR}" ]; then
-            mkdir -p ${DATA_EXCHANGE_DIR}
-          fi
+          mkdir -p ${DATA_EXCHANGE_DIR}
           DATA_EXCHANGE_MOUNT_POINT=/tmp/exchange  # Exchange folder inside of the container
           docker_args+=(--volume=${DATA_EXCHANGE_DIR}:${DATA_EXCHANGE_MOUNT_POINT})
           script_args+=(--exchange-dir ${DATA_EXCHANGE_MOUNT_POINT})
@@ -196,13 +211,28 @@ jobs:
         HOST_CONFIG=${{ inputs.HOST_CONFIG }}
         script_args+=(${HOST_CONFIG:+"--host-config ${HOST_CONFIG}"})
 
-        if ${{ inputs.USE_SCCACHE }} == 'true'; then
-          script_args+=(--sccache-credentials $(basename ${GOOGLE_GHA_CREDS_PATH}))
+        if [[ "${{ inputs.USE_SCCACHE }}" == "true" ]]; then
+          case "${{ inputs.RUNS_ON }}" in
+            streak2*)
+              SCCACHE_HOST_DIR="${{ inputs.SCCACHE_HOST_DIR }}"
+              if [[ -z "${SCCACHE_HOST_DIR}" ]]; then
+                SCCACHE_HOST_DIR=/home/geos-data/sccache
+              fi
+
+              mkdir -p "${SCCACHE_HOST_DIR}"
+              docker_args+=(--volume=${SCCACHE_HOST_DIR}:/sccache)
+              docker_args+=(-e SCCACHE_DIR=/sccache)
+              if [[ -n "${SCCACHE_CACHE_SIZE:-}" ]]; then
+                docker_args+=(-e SCCACHE_CACHE_SIZE)
+              fi
+              script_args+=(--use-sccache)
+              ;;
+            *)
+              script_args+=(--sccache-credentials $(basename ${GOOGLE_GHA_CREDS_PATH}))
+              ;;
+          esac
         fi
 
-        # We need to know where the code folder is mounted inside the container so we can run the script at the proper location!
-        # Since this information is repeated twice, we use a variable.
-        GITHUB_WORKSPACE_MOUNT_POINT=/tmp/geos
         docker_args+=(--volume=${GITHUB_WORKSPACE}:${GITHUB_WORKSPACE_MOUNT_POINT})
         script_args+=(--repository ${GITHUB_WORKSPACE_MOUNT_POINT})
 
@@ -327,48 +357,53 @@ jobs:
         echo "Received exit status ${EXIT_STATUS} from the build process."
         set -e
 
-        # Send to the bucket and print the download link when it makes sense.
-        if [[ ! -z "${{ inputs.GCP_BUCKET }}" ]]; then
-          if [[ "${{ inputs.BUILD_TYPE }}" = "integrated_tests" || ${EXIT_STATUS} -eq 0 ]]; then
-            # disable warning about parallel composite upload
-            CLOUDSDK_PYTHON=python3 gcloud config set storage/parallel_composite_upload_enabled True
-
-            if [ -f ${DATA_EXCHANGE_DIR}/${DATA_BASENAME} ]; then
-              CLOUDSDK_PYTHON=python3 gcloud storage cp --quiet -a publicRead ${DATA_EXCHANGE_DIR}/${DATA_BASENAME} gs://${{ inputs.GCP_BUCKET }}/
-              echo "Download the bundle at https://storage.googleapis.com/${{ inputs.GCP_BUCKET }}/${DATA_BASENAME}"
-            fi
+        if [[ -n "${DATA_BASENAME}" ]]; then
+          if [ -f "${DATA_EXCHANGE_DIR}/${DATA_BASENAME}" ]; then
+            echo "Generated artifact at ${DATA_EXCHANGE_DIR}/${DATA_BASENAME}"
+          fi
 
-            if [ -f ${DATA_EXCHANGE_DIR}/test_logs_${DATA_BASENAME} ]; then
-              CLOUDSDK_PYTHON=python3 gcloud storage cp --quiet -a publicRead ${DATA_EXCHANGE_DIR}/test_logs_${DATA_BASENAME} gs://${{ inputs.GCP_BUCKET }}/
-              echo "Download integrated test logs here: https://storage.googleapis.com/${{ inputs.GCP_BUCKET }}/test_logs_${DATA_BASENAME}"
-            fi
+          if [ -f "${DATA_EXCHANGE_DIR}/test_logs_${DATA_BASENAME}" ]; then
+            echo "Generated integrated test logs at ${DATA_EXCHANGE_DIR}/test_logs_${DATA_BASENAME}"
+          fi
 
-            if [ -f ${DATA_EXCHANGE_DIR}/baseline_${DATA_BASENAME} ];then
+          if [[ -n "${{ inputs.LOCAL_BASELINE_DIR }}" ]]; then
+            THIS_PR_NUMBER=pr${{ github.event.number }}
+            TARGET_DIR="${{ inputs.LOCAL_BASELINE_DIR }}/baselines_${THIS_PR_NUMBER}"
+            mkdir -p "${TARGET_DIR}"
+
+            for file in \
+              "${DATA_EXCHANGE_DIR}/baseline_${DATA_BASENAME}" \
+              "${DATA_EXCHANGE_DIR}/test_logs_${DATA_BASENAME}"
+            do
+              if [ -f "${file}" ]; then
+                cp "${file}" "${TARGET_DIR}"
+                echo "Copied $(basename "${file}") to ${TARGET_DIR}"
+              fi
+            done
+          fi
 
-              if [[ -n "${{ inputs.LOCAL_BASELINE_DIR }}" ]]; then
-                # 1. We copy the baselines to a local directory to store them
+          if [ ! -f "${DATA_EXCHANGE_DIR}/baseline_${DATA_BASENAME}" ]; then
+            echo "No new baseline archive was generated."
+          fi
+        fi
 
-                # 1.a Create the new target directory to store the new baselines
-                THIS_PR_NUMBER=pr${{ github.event.number }}
-                NEW_PR_BASELINE_FOLDER_NAME=baselines_${THIS_PR_NUMBER}
-                TARGET_DIR="${{ inputs.LOCAL_BASELINE_DIR }}/${NEW_PR_BASELINE_FOLDER_NAME}"
-                echo "Create folder ${TARGET_DIR}"
-                mkdir -p "${TARGET_DIR}"
+        if [[ "${{ inputs.UPLOAD_TO_GCP }}" == "true" && -n "${{ inputs.GCP_BUCKET }}" ]]; then
+          if [[ "${{ inputs.BUILD_TYPE }}" = "integrated_tests" || ${EXIT_STATUS} -eq 0 ]]; then
+            CLOUDSDK_PYTHON=python3 gcloud config set storage/parallel_composite_upload_enabled True
 
-                # 1.b We copy the new baselines to the new target directory
-                SOURCE_FILE="${DATA_EXCHANGE_DIR}/baseline_${DATA_BASENAME}"
-                echo "Copy ${SOURCE_FILE} to ${TARGET_DIR}"
-                cp "${SOURCE_FILE}" "${TARGET_DIR}"
+            for file in \
+              "${DATA_EXCHANGE_DIR}/${DATA_BASENAME}" \
+              "${DATA_EXCHANGE_DIR}/test_logs_${DATA_BASENAME}" \
+              "${DATA_EXCHANGE_DIR}/baseline_${DATA_BASENAME}"
+            do
+              if [ -f "${file}" ]; then
+                CLOUDSDK_PYTHON=python3 gcloud storage cp --quiet -a publicRead "${file}" gs://${{ inputs.GCP_BUCKET }}/
+                echo "Uploaded https://storage.googleapis.com/${{ inputs.GCP_BUCKET }}/$(basename "${file}")"
               fi
-
-              # 2. We push the baselines to the cloud
-              CLOUDSDK_PYTHON=python3 gcloud storage cp --quiet -a publicRead ${DATA_EXCHANGE_DIR}/baseline_${DATA_BASENAME} gs://${{ inputs.GCP_BUCKET }}/
-              echo "Download test baselines here: https://storage.googleapis.com/${{ inputs.GCP_BUCKET }}/baseline_${DATA_BASENAME}"
-              echo "New baseline ID: baseline_${DATA_BASENAME::-7}"
-            else
-              echo "Baselines ${DATA_EXCHANGE_DIR}/baseline_${DATA_BASENAME} were not uploaded. Likeyly because no rebaseline was necessary."
-            fi
+            done
           fi
+        else
+          echo "GCP upload disabled."
         fi
 
         exit ${EXIT_STATUS}
@@ -402,6 +437,16 @@ jobs:
         REQUESTS_CA_BUNDLE: /etc/pki/tls/certs/ca-bundle.crt
         CURL_CA_BUNDLE: /etc/pki/tls/certs/ca-bundle.crt
 
+    - name: Show sccache summary
+      if: ${{ always() && inputs.USE_SCCACHE }}
+      run: |
+        SCCACHE_STATS_FILE="${GITHUB_WORKSPACE}/.sccache-runtime/stats.txt"
+        if [[ -f "${SCCACHE_STATS_FILE}" ]]; then
+          grep -E '^(Compile requests|Cache hits|Cache misses|Cache timeouts|Cache read errors|Cache write errors|Compilation failures|Cache errors|Non-cacheable compilations|Non-cacheable calls|Cache location|Cache size|Max cache size|Version \(client\))' "${SCCACHE_STATS_FILE}" || cat "${SCCACHE_STATS_FILE}"
+        else
+          echo "No in-container sccache stats file was captured."
+        fi
+
     - name: 'Cleanup build folder'
       if: always()
       run: |

.github/workflows/ci_tests.yml

@@ -195,6 +195,9 @@ jobs:
             ENABLE_TRILINOS: OFF
             BUILD_SHARED_LIBS: ON
             GEOS_ENABLE_BOUNDS_CHECK: OFF
+            RUNS_ON: streak2
+            NPROC: 8
+            DOCKER_RUN_ARGS: "--cpus=8 --memory=192g -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.crt:/certs/ca-bundle.crt:ro"
             HOST_CONFIG: /spack-generated.cmake
 
           - name: Rocky Linux 8 - gcc 12
@@ -223,6 +226,9 @@ jobs:
             ENABLE_TRILINOS: OFF
             BUILD_SHARED_LIBS: ON
             GEOS_ENABLE_BOUNDS_CHECK: ON
+            RUNS_ON: streak2
+            NPROC: 8
+            DOCKER_RUN_ARGS: "--cpus=8 --memory=192g -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.crt:/certs/ca-bundle.crt:ro"
             HOST_CONFIG: /spack-generated.cmake
 
           - name: Rocky Linux 9 - clang 22
@@ -233,6 +239,9 @@ jobs:
             ENABLE_TRILINOS: OFF
             BUILD_SHARED_LIBS: ON
             GEOS_ENABLE_BOUNDS_CHECK: ON
+            RUNS_ON: streak2
+            NPROC: 8
+            DOCKER_RUN_ARGS: "--cpus=8 --memory=192g -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.crt:/certs/ca-bundle.crt:ro"
             HOST_CONFIG: /spack-generated.cmake
 
     uses: ./.github/workflows/build_and_test.yml
@@ -247,8 +256,10 @@ jobs:
       ENABLE_TRILINOS: ${{ matrix.ENABLE_TRILINOS }}
       GEOS_ENABLE_BOUNDS_CHECK: ${{ matrix.GEOS_ENABLE_BOUNDS_CHECK }}
       GCP_BUCKET: ${{ matrix.GCP_BUCKET }}
+      UPLOAD_TO_GCP: ${{ matrix.UPLOAD_TO_GCP || false }}
       HOST_CONFIG: ${{ matrix.HOST_CONFIG }}
       NPROC: ${{ matrix.NPROC || '' }}
+      CTEST_PARALLEL_LEVEL: ${{ matrix.CTEST_PARALLEL_LEVEL || matrix.NPROC || '' }}
       RUNS_ON: ${{ matrix.RUNS_ON || 'ubuntu-22.04' }}
     secrets: inherit
 
@@ -270,11 +281,12 @@ jobs:
       ENABLE_TRILINOS: OFF
       GEOS_ENABLE_BOUNDS_CHECK: ON
       GCP_BUCKET: geosx/integratedTests
+      UPLOAD_TO_GCP: false
       RUNS_ON: streak2-32core
       NPROC: 32
       DOCKER_RUN_ARGS: "--cpus=32 --memory=256g -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.crt:/certs/ca-bundle.crt:ro"
       REQUIRED_LABEL: "ci: run integrated tests"
-      LOCAL_BASELINE_DIR: /data/GEOS/baselines
+      LOCAL_BASELINE_DIR: /home/geos-data/baselines
       HOST_CONFIG: /spack-generated.cmake
 
   baseline_log:
@@ -422,6 +434,7 @@ jobs:
       GEOS_ENABLE_BOUNDS_CHECK: ${{ matrix.GEOS_ENABLE_BOUNDS_CHECK }}
       HOST_CONFIG: ${{ matrix.HOST_CONFIG }}
       NPROC: ${{ matrix.NPROC }}
+      CTEST_PARALLEL_LEVEL: ${{ matrix.CTEST_PARALLEL_LEVEL || matrix.NPROC }}
       RUNS_ON: ${{ matrix.RUNS_ON }}
       REQUIRED_LABEL: "ci: run CUDA builds"
     secrets: inherit

inputFiles/compositionalMultiphaseWell/benchmarks/Egg/compositionalMultiphaseWell_Egg.ats

@@ -12,8 +12,8 @@ decks = [
         partitions=[
             (1, 1, 1),
         ],
-        restart_step=20,
-        check_step=35,
+        restart_step=1,
+        check_step=3,
         restartcheck_params=RestartcheckParameters(**restartcheck_params)),
 ]