diff --git a/.circleci/images/primary/Dockerfile b/.circleci/images/primary/Dockerfile index dd800415252..762c46abbe9 100644 --- a/.circleci/images/primary/Dockerfile +++ b/.circleci/images/primary/Dockerfile @@ -1,4 +1,4 @@ -FROM circleci/buildpack-deps:bionic +FROM circleci/buildpack-deps:bionic@sha256:5ecd6d9cb3e0a74a2563a48baee224b6d79341638dc843d9072b4f08cf23be43 RUN sudo dpkg --add-architecture i386 RUN sudo apt-get -y -qq update diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000000..ff94907c808 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,16 @@ +version: 2 +updates: + - package-ecosystem: docker + directory: /.circleci/images/primary + schedule: + interval: daily + + - package-ecosystem: github-actions + directory: / + schedule: + interval: daily + + - package-ecosystem: docker + directory: /contrib/docker + schedule: + interval: daily diff --git a/.github/workflows/dev-long-tests.yml b/.github/workflows/dev-long-tests.yml index bc88c4a6a1f..3c766a0b109 100644 --- a/.github/workflows/dev-long-tests.yml +++ b/.github/workflows/dev-long-tests.yml @@ -16,42 +16,72 @@ jobs: DEVNULLRIGHTS: 1 READFROMBLOCKDEVICE: 1 steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: make test run: make test make-test-osx: runs-on: macos-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: OS-X test run: make test # make -c lib all doesn't work because of the fact that it's not a tty no-intrinsics-fuzztest: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: no intrinsics fuzztest run: MOREFLAGS="-DZSTD_NO_INTRINSICS" make -C tests fuzztest tsan-zstreamtest: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: thread sanitizer zstreamtest run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream tsan-fuzztest: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: thread sanitizer fuzztest run: CC=clang make tsan-fuzztest gcc-8-asan-ubsan-testzstd: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: gcc-8 + ASan + UBSan + Test Zstd run: | sudo apt-get -qqq update @@ -61,7 +91,12 @@ jobs: gcc-asan-ubsan-testzstd-32bit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: ASan + UBSan + Test Zstd, 32bit mode run: | make libc6install @@ -74,7 +109,12 @@ jobs: gcc-8-asan-ubsan-fuzz: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: gcc-8 + ASan + UBSan + Fuzz Test run: | make gcc8install @@ -83,7 +123,12 @@ jobs: gcc-asan-ubsan-fuzz32: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: ASan + UBSan + Fuzz Test 32bit run: | make libc6install @@ -92,21 +137,36 @@ jobs: asan-ubsan-regression: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: ASan + UBSan + Regression Test run: make -j uasanregressiontest msan-regression: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: MSan + Regression Test run: make -j msanregressiontest clang-msan-fuzz: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: clang + MSan + Fuzz Test run: | sudo apt-get -qqq update @@ -116,7 +176,12 @@ jobs: clang-msan-testzstd: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: clang + MSan + Test Zstd run: | sudo apt-get update @@ -126,7 +191,12 @@ jobs: armfuzz: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: Qemu ARM emulation + Fuzz Test run: | sudo apt-get -qqq update @@ -136,7 +206,12 @@ jobs: valgrind-fuzz-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: valgrind + fuzz test stack mode # ~ 7mn shell: 'script -q -e -c "bash {0}"' run: | @@ -152,22 +227,27 @@ jobs: matrix: sanitizer: [address, undefined, memory] steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + - name: Build Fuzzers (${{ matrix.sanitizer }}) id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@17edda86ca7183dcb5dddabb2f6cec44a7cbc2a0 # master with: oss-fuzz-project-name: 'zstd' dry-run: false sanitizer: ${{ matrix.sanitizer }} - name: Run Fuzzers (${{ matrix.sanitizer }}) - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@17edda86ca7183dcb5dddabb2f6cec44a7cbc2a0 # master with: oss-fuzz-project-name: 'zstd' fuzz-seconds: 600 dry-run: false sanitizer: ${{ matrix.sanitizer }} - name: Upload Crash - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@3446296876d12d4e3a0f3145a3c87e67bf0a16b5 # v1.0.0 if: failure() && steps.build.outcome == 'success' with: name: ${{ matrix.sanitizer }}-artifacts diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml index 0092775b1a7..68252947f14 100644 --- a/.github/workflows/dev-short-tests.yml +++ b/.github/workflows/dev-short-tests.yml @@ -14,21 +14,36 @@ jobs: linux-kernel: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: linux kernel, library + build + test run: make -C contrib/linux-kernel test CFLAGS="-Werror -Wunused-const-variable -Wunused-but-set-variable" benchmarking: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: make benchmarking run: make benchmarking check-32bit: # designed to catch https://github.com/facebook/zstd/issues/2428 runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: make check on 32-bit run: | sudo apt update @@ -38,7 +53,12 @@ jobs: gcc-7-libzstd: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: gcc-7 + libzstdmt compilation run: | sudo apt-get -qqq update @@ -53,7 +73,12 @@ jobs: cmake-build-and-test-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: cmake build and test check run: | FUZZERTEST=-T1mn ZSTREAM_TESTTIME=-T1mn make cmakebuild @@ -64,7 +89,12 @@ jobs: cpp-gnu90-c99-compatibility: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: C++, gnu90 and c99 compatibility run: | make cxxtest @@ -78,7 +108,12 @@ jobs: mingw-cross-compilation: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: mingw cross-compilation run: | # sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; (doesn't work) @@ -89,7 +124,12 @@ jobs: armbuild: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: ARM Build Test run: | sudo apt-get -qqq update @@ -99,7 +139,12 @@ jobs: bourne-shell: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: Bourne shell compatibility (shellcheck) run: | wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz @@ -109,7 +154,12 @@ jobs: zlib-wrapper: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: zlib wrapper test run: | sudo apt-get -qqq update @@ -120,7 +170,12 @@ jobs: lz4-threadpool-partial-libs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: LZ4, thread pool, and partial libs testslib wrapper test run: | make lz4install @@ -134,7 +189,12 @@ jobs: gcc-make-tests-32bit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: Make all, 32bit mode run: | sudo apt-get -qqq update @@ -144,7 +204,12 @@ jobs: gcc-8-make: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: gcc-8 build run: | sudo apt-get -qqq update @@ -159,9 +224,14 @@ jobs: platform: [x64, Win32] configuration: [Debug, Release] steps: - - uses: actions/checkout@v2 + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: Add MSBuild to PATH - uses: microsoft/setup-msbuild@v1.0.2 + uses: microsoft/setup-msbuild@c26a08ba26249b81327e26f6ef381897b6a8754d # v1.0.2 - name: Build working-directory: ${{env.GITHUB_WORKSPACE}} run: > @@ -186,7 +256,7 @@ jobs: # sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main" # sudo apt-get update # sudo apt-get install -y intel-basekit intel-hpckit -# - uses: actions/checkout@v2 +# - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 # - name: make check # run: | # make CC=/opt/intel/oneapi/compiler/latest/linux/bin/intel64/icc check diff --git a/.github/workflows/publish-release-artifacts.yml b/.github/workflows/publish-release-artifacts.yml index 952cb26b7e9..7931413ab63 100644 --- a/.github/workflows/publish-release-artifacts.yml +++ b/.github/workflows/publish-release-artifacts.yml @@ -11,8 +11,13 @@ jobs: if: startsWith(github.ref, 'refs/tags/') steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 + with: + egress-policy: audit + - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: Archive env: @@ -61,7 +66,7 @@ jobs: fi - name: Publish - uses: skx/github-action-publish-binaries@release-1.3 + uses: skx/github-action-publish-binaries@9f2ea568baec5017a7d608f03ef143a57745cc5c # release-1.3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000000..d8562a434e2 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,14 @@ +repos: +- repo: https://github.com/gitleaks/gitleaks + rev: v8.16.3 + hooks: + - id: gitleaks +- repo: https://github.com/jumanjihouse/pre-commit-hooks + rev: 3.0.0 + hooks: + - id: shellcheck +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace