diff --git a/.github/workflows/test-lang-csharp.yml b/.github/workflows/test-lang-csharp.yml index 81fafce1940..29c812edaec 100644 --- a/.github/workflows/test-lang-csharp.yml +++ b/.github/workflows/test-lang-csharp.yml @@ -34,6 +34,10 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Add libzstd + shell: bash + run: sudo apt-get install -y libzstd-dev + - name: Install .NET SDKs uses: actions/setup-dotnet@v1 with: @@ -60,6 +64,10 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Add libzstd + shell: bash + run: sudo apt-get install -y libzstd-dev + - name: Install .NET SDKs uses: actions/setup-dotnet@v1 with: diff --git a/.github/workflows/test-lang-java.yml b/.github/workflows/test-lang-java.yml index 663f6f6dfb5..0daa77c8127 100644 --- a/.github/workflows/test-lang-java.yml +++ b/.github/workflows/test-lang-java.yml @@ -104,6 +104,14 @@ jobs: python3 -m pip install --upgrade pip setuptools tox-wheel python3 -m pip install python-snappy zstandard + - name: Setup C# for Generating Interop Data + uses: actions/setup-dotnet@v1 + with: + dotnet-version: | + 3.1.x + 5.0.x + 6.0.x + - name: Install Java Avro for Interop Test working-directory: . run: mvn -B install -DskipTests @@ -120,6 +128,10 @@ jobs: working-directory: lang/py run: ./build.sh interop-data-generate + - name: Generate Interop Data using C# + working-directory: lang/csharp + run: ./build.sh interop-data-generate + - name: Run Interop Tests working-directory: lang/java/ipc run: mvn -B test -P interop-data-test diff --git a/.travis.yml b/.travis.yml index a9967017447..8668fe638b0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -94,11 +94,12 @@ matrix: update: true packages: - wget + - libzstd-dev install: - wget https://dot.net/v1/dotnet-install.sh - - bash ./dotnet-install.sh --version "3.1.402" --install-dir "$HOME/.dotnet" --architecture "arm64" --os "linux" # 3.1 - - bash ./dotnet-install.sh --version "5.0.101" --install-dir "$HOME/.dotnet" --architecture "arm64" --os "linux" # 5.0 - - bash ./dotnet-install.sh --channel Current # 6.0 + - bash ./dotnet-install.sh --channel "3.1" --install-dir "$HOME/.dotnet" # 3.1 + - bash ./dotnet-install.sh --channel "5.0" --install-dir "$HOME/.dotnet" # 5.0 + - bash ./dotnet-install.sh --channel "6.0" --install-dir "$HOME/.dotnet" # 6.0 before_script: - export PATH=$HOME/.dotnet:$PATH - dotnet --list-sdks diff --git a/lang/csharp/Avro.sln b/lang/csharp/Avro.sln index ec3d559be73..b945f4ead82 100644 --- a/lang/csharp/Avro.sln +++ b/lang/csharp/Avro.sln @@ -13,6 +13,24 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.msbuild", "src\apache\ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.perf", "src\apache\perf\Avro.perf.csproj", "{AC4E1909-2594-4D01-9B2B-B832C07BAFE5}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Codecs", "Codecs", "{0FAEE4F6-D72F-4B18-869A-7A90BAC1280F}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.BZip2", "src\apache\codec\Avro.File.BZip2\Avro.File.BZip2.csproj", "{FFA119B2-0D60-4090-B5A6-ECA718138812}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.BZip2.Test", "src\apache\codec\Avro.File.BZip2.Test\Avro.File.BZip2.Test.csproj", "{D5ED6642-3E33-493F-9217-FE00E4885699}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.Snappy", "src\apache\codec\Avro.File.Snappy\Avro.File.Snappy.csproj", "{B15BEEDC-A371-46D0-BFF6-63FC8105B520}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.Snappy.Test", "src\apache\codec\Avro.File.Snappy.Test\Avro.File.Snappy.Test.csproj", "{AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.XZ", "src\apache\codec\Avro.File.XZ\Avro.File.XZ.csproj", "{98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.XZ.Test", "src\apache\codec\Avro.File.XZ.Test\Avro.File.XZ.Test.csproj", "{99711F8E-C5C1-4864-A51F-3317E19CAD7B}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.Zstandard", "src\apache\codec\Avro.File.Zstandard\Avro.File.Zstandard.csproj", "{8207A628-6285-4DDF-B846-C0C7ED3E3D16}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.Zstandard.Test", "src\apache\codec\Avro.File.Zstandard.Test\Avro.File.Zstandard.Test.csproj", "{04264DDD-C204-4F59-88D4-FB4C69BD80C3}" +EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{8A671DF3-BC71-4E1A-BB06-0A225799A274}" ProjectSection(SolutionItems) = preProject ..\..\.editorconfig = ..\..\.editorconfig @@ -89,10 +107,116 @@ Global {AC4E1909-2594-4D01-9B2B-B832C07BAFE5}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU {AC4E1909-2594-4D01-9B2B-B832C07BAFE5}.Release|Mixed Platforms.Build.0 = Release|Any CPU {AC4E1909-2594-4D01-9B2B-B832C07BAFE5}.Release|x86.ActiveCfg = Release|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Debug|x86.ActiveCfg = Debug|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Debug|x86.Build.0 = Debug|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Release|Any CPU.Build.0 = Release|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Release|x86.ActiveCfg = Release|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Release|x86.Build.0 = Release|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Debug|x86.ActiveCfg = Debug|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Debug|x86.Build.0 = Debug|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Release|Any CPU.Build.0 = Release|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Release|x86.ActiveCfg = Release|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Release|x86.Build.0 = Release|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Debug|x86.ActiveCfg = Debug|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Debug|x86.Build.0 = Debug|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Release|Any CPU.Build.0 = Release|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Release|x86.ActiveCfg = Release|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Release|x86.Build.0 = Release|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Debug|x86.ActiveCfg = Debug|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Debug|x86.Build.0 = Debug|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Release|Any CPU.Build.0 = Release|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Release|x86.ActiveCfg = Release|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Release|x86.Build.0 = Release|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Debug|x86.ActiveCfg = Debug|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Debug|x86.Build.0 = Debug|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Release|Any CPU.Build.0 = Release|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Release|x86.ActiveCfg = Release|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Release|x86.Build.0 = Release|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Debug|x86.ActiveCfg = Debug|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Debug|x86.Build.0 = Debug|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Release|Any CPU.Build.0 = Release|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Release|x86.ActiveCfg = Release|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Release|x86.Build.0 = Release|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Debug|x86.ActiveCfg = Debug|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Debug|x86.Build.0 = Debug|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Release|Any CPU.Build.0 = Release|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Release|x86.ActiveCfg = Release|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Release|x86.Build.0 = Release|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Debug|x86.ActiveCfg = Debug|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Debug|x86.Build.0 = Debug|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Release|Any CPU.Build.0 = Release|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Release|x86.ActiveCfg = Release|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {FFA119B2-0D60-4090-B5A6-ECA718138812} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F} + {D5ED6642-3E33-493F-9217-FE00E4885699} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F} + {B15BEEDC-A371-46D0-BFF6-63FC8105B520} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F} + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F} + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F} + {99711F8E-C5C1-4864-A51F-3317E19CAD7B} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F} + {8207A628-6285-4DDF-B846-C0C7ED3E3D16} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F} + {04264DDD-C204-4F59-88D4-FB4C69BD80C3} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F} + EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {ACE75CE8-16B2-4C6E-A5BE-B6F6DB5FE095} EndGlobalSection diff --git a/lang/csharp/README.md b/lang/csharp/README.md index e2050c035a1..8d829c0b777 100644 --- a/lang/csharp/README.md +++ b/lang/csharp/README.md @@ -17,15 +17,19 @@ Install-Package Apache.Avro ## Project Target Frameworks -| Project | Type | .NET Standard 2.0 | .NET Standard 2.1 | .NET Core 3.1 | .NET 5.0 | .NET 6.0 | -|:---------------:|:----------:|:------------------:|:-----------------:|:-------------:|:---------:|:---------:| -| Avro.codegen | Exe | | | ✔️ |✔️ |✔️ | -| Avro.ipc | Library | ✔️ | ✔️ | | | | -| Avro.ipc.test | Unit Tests | | | ✔️ |✔️ |✔️ | -| Avro.main | Library | ✔️ | ✔️ | | | | -| Avro.msbuild | Library | ✔️ | ✔️ | | | | -| Avro.perf | Exe | | | ✔️ |✔️ |✔️ | -| Avro.test | Unit Tests | | | ✔️ |✔️ |✔️ | +| Project | Published to nuget.org | Type | .NET Standard 2.0 | .NET Standard 2.1 | .NET Core 3.1 | .NET 5.0 | .NET 6.0 | +|:-------------------:|:--------------------------:|:----------:|:------------------:|:-----------------:|:-------------:|:---------:|:---------:| +| Avro.main | Apache.Avro | Library | ✔️ | ✔️ | | | | +| Avro.File.Snappy | Apache.Avro.File.Snappy | Library | ✔️ | ✔️ | | | | +| Avro.File.BZip2 | Apache.Avro.File.BZip2 | Library | ✔️ | ✔️ | | | | +| Avro.File.XZ | Apache.Avro.File.XZ | Library | ✔️ | ✔️ | | | | +| Avro.File.Zstandard | Apache.Avro.File.Zstandard | Library | ✔️ | ✔️ | | | | +| Avro.codegen | Apache.Avro.Tools | Exe | | | ✔️ |✔️ |✔️ | +| Avro.ipc | | Library | ✔️ | ✔️ | | | | +| Avro.ipc.test | | Unit Tests | | | ✔️ |✔️ |✔️ | +| Avro.msbuild | | Library | ✔️ | ✔️ | | | | +| Avro.perf | | Exe | | | ✔️ |✔️ |✔️ | +| Avro.test | | Unit Tests | | | ✔️ |✔️ |✔️ | ## Dependency package version strategy diff --git a/lang/csharp/build.sh b/lang/csharp/build.sh index 6fe5cdf87a9..5a1c1d81dca 100755 --- a/lang/csharp/build.sh +++ b/lang/csharp/build.sh @@ -36,7 +36,7 @@ do dotnet build --configuration Release Avro.sln # AVRO-2442: Explicitly set LANG to work around ICU bug in `dotnet test` - LANG=en_US.UTF-8 dotnet test --configuration Release --no-build \ + LANG=en_US.UTF-8 dotnet test --configuration Release --no-build \ --filter "TestCategory!=Interop" Avro.sln ;; @@ -50,13 +50,20 @@ do dotnet pack --configuration Release Avro.sln # add the binary LICENSE and NOTICE to the tarball - mkdir build/ + mkdir -p build/ cp LICENSE NOTICE build/ # add binaries to the tarball - mkdir build/main/ + mkdir -p build/main/ cp -R src/apache/main/bin/Release/* build/main/ - mkdir build/codegen/ + # add codec binaries to the tarball + for codec in Avro.File.Snappy Avro.File.BZip2 Avro.File.XZ Avro.File.Zstandard + do + mkdir -p build/codec/$codec/ + cp -R src/apache/codec/$codec/bin/Release/* build/codec/$codec/ + done + # add codegen binaries to the tarball + mkdir -p build/codegen/ cp -R src/apache/codegen/bin/Release/* build/codegen/ # build the tarball @@ -74,7 +81,7 @@ do ;; interop-data-test) - LANG=en_US.UTF-8 dotnet test --filter "TestCategory=Interop" --verbosity normal + LANG=en_US.UTF-8 dotnet test --filter "TestCategory=Interop" --logger "console;verbosity=normal;noprogress=true" src/apache/test/Avro.test.csproj ;; clean) diff --git a/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/Avro.File.BZip2.Test.csproj b/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/Avro.File.BZip2.Test.csproj new file mode 100644 index 00000000000..ace1db23c2c --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/Avro.File.BZip2.Test.csproj @@ -0,0 +1,42 @@ + + + + + + + $(DefaultUnitTestTargetFrameworks) + false + + + + true + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/BZip2Tests.cs b/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/BZip2Tests.cs new file mode 100644 index 00000000000..821cb4b7233 --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/BZip2Tests.cs @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using System.IO; +using System.Linq; +using NUnit.Framework; + +namespace Avro.File.BZip2.Test +{ + public class Tests + { + private static readonly int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 100000 }; + + [Test, Combinatorial] + public void CompressDecompress([ValueSource(nameof(_testLengths))] int length, [Values] BZip2Level level) + { + byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); + + BZip2Codec codec = new BZip2Codec(level); + + byte[] compressed = codec.Compress(data); + byte[] uncompressed = codec.Decompress(compressed, compressed.Length); + + Assert.IsTrue(Enumerable.SequenceEqual(data, uncompressed)); + } + + [Test, Combinatorial] + public void CompressDecompressStream([ValueSource(nameof(_testLengths))] int length, [Values] BZip2Level level) + { + byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); + + BZip2Codec codec = new BZip2Codec(level); + + using (MemoryStream inputStream = new MemoryStream(data)) + using (MemoryStream outputStream = new MemoryStream()) + { + codec.Compress(inputStream, outputStream); + + byte[] compressed = outputStream.ToArray(); + byte[] uncompressed = codec.Decompress(compressed, compressed.Length); + + Assert.IsTrue(Enumerable.SequenceEqual(data, uncompressed)); + } + } + + [Test] + public void ToStringAndName([Values] BZip2Level level) + { + BZip2Codec codec = new BZip2Codec(level); + + Assert.AreEqual("bzip2", codec.GetName()); + Assert.AreEqual($"bzip2-{(int)level}", codec.ToString()); + } + + [Test] + public void DefaultLevel() + { + BZip2Codec codec = new BZip2Codec(); + + Assert.AreEqual(BZip2Level.Default, codec.Level); + } + + [Test] + public void Equal([Values] BZip2Level level) + { + BZip2Codec codec1 = new BZip2Codec(level); + BZip2Codec codec2 = new BZip2Codec(level); + + Assert.IsTrue(codec1.Equals(codec1)); + Assert.IsTrue(codec2.Equals(codec2)); + Assert.IsTrue(codec1.Equals(codec2)); + Assert.IsTrue(codec2.Equals(codec1)); + } + + [Test] + public void HashCode([Values] BZip2Level level) + { + BZip2Codec codec = new BZip2Codec(level); + + Assert.AreNotEqual(0, codec.GetHashCode()); + } + } +} diff --git a/lang/csharp/src/apache/codec/Avro.File.BZip2/Avro.File.BZip2.csproj b/lang/csharp/src/apache/codec/Avro.File.BZip2/Avro.File.BZip2.csproj new file mode 100644 index 00000000000..8dac7c9f1ed --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.File.BZip2/Avro.File.BZip2.csproj @@ -0,0 +1,47 @@ + + + + + + + $(DefaultLibraryTargetFrameworks) + Avro.File.BZip2 + true + ../../../../Avro.snk + + + + + Apache.Avro.File.BZip2 + BZip2 compression library for Apache.Avro + + + + true + + + + + + + + + + + + diff --git a/lang/csharp/src/apache/codec/Avro.File.BZip2/BZip2.cs b/lang/csharp/src/apache/codec/Avro.File.BZip2/BZip2.cs new file mode 100644 index 00000000000..354c39cc89e --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.File.BZip2/BZip2.cs @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using System.IO; + +namespace Avro.File.BZip2 +{ + /// + /// BZip2 Compression level + /// + public enum BZip2Level + { + Default = 9, + Level1 = 1, + Level2 = 2, + Level3 = 3, + Level4 = 4, + Level5 = 5, + Level6 = 6, + Level7 = 7, + Level8 = 8, + Level9 = 9 + } + + /// + /// Implements BZip2 compression and decompression. + /// + public class BZip2Codec : Codec + { + public BZip2Level Level {get; private set;} + + public BZip2Codec() + : this(BZip2Level.Default) + { + } + + public BZip2Codec(BZip2Level level) + { + Level = level; + } + + /// + public override byte[] Compress(byte[] uncompressedData) + { + using (MemoryStream inputStream = new MemoryStream(uncompressedData)) + using (MemoryStream outputStream = new MemoryStream()) + { + Compress(inputStream, outputStream); + return outputStream.ToArray(); + } + } + + /// + public override void Compress(MemoryStream inputStream, MemoryStream outputStream) + { + inputStream.Position = 0; + outputStream.SetLength(0); + ICSharpCode.SharpZipLib.BZip2.BZip2.Compress(inputStream, outputStream, false, (int)Level); + } + + /// + public override byte[] Decompress(byte[] compressedData, int blockLength) + { + using (MemoryStream inputStream = new MemoryStream(compressedData, 0, blockLength)) + using (MemoryStream outputStream = new MemoryStream()) + { + ICSharpCode.SharpZipLib.BZip2.BZip2.Decompress(inputStream, outputStream, false); + return outputStream.ToArray(); + } + } + + /// + public override string GetName() + { + return DataFileConstants.BZip2Codec; + } + + /// + public override bool Equals(object other) + { + return this == other || GetType().Name == other.GetType().Name; + } + + /// + public override int GetHashCode() + { + return GetName().GetHashCode(); + } + + /// + public override string ToString() + { + return $"{GetName()}-{(int)Level}"; + } + } +} diff --git a/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/Avro.File.Snappy.Test.csproj b/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/Avro.File.Snappy.Test.csproj new file mode 100644 index 00000000000..ab325b0fa64 --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/Avro.File.Snappy.Test.csproj @@ -0,0 +1,42 @@ + + + + + + + $(DefaultUnitTestTargetFrameworks) + false + + + + true + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/SnappyTests.cs b/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/SnappyTests.cs new file mode 100644 index 00000000000..148d493aae2 --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/SnappyTests.cs @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using System.IO; +using System.Linq; +using NUnit.Framework; + +namespace Avro.File.Snappy.Test +{ + public class Tests + { + private static readonly int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 100000 }; + + [TestCaseSource(nameof(_testLengths))] + public void CompressDecompress(int length) + { + byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); + + SnappyCodec codec = new SnappyCodec(); + + byte[] compressed = codec.Compress(data); + byte[] uncompressed = codec.Decompress(compressed, compressed.Length); + + Assert.IsTrue(Enumerable.SequenceEqual(data, uncompressed)); + } + + [TestCaseSource(nameof(_testLengths))] + public void CompressDecompressStream(int length) + { + byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); + + SnappyCodec codec = new SnappyCodec(); + + using (MemoryStream inputStream = new MemoryStream(data)) + using (MemoryStream outputStream = new MemoryStream()) + { + codec.Compress(inputStream, outputStream); + + byte[] compressed = outputStream.ToArray(); + byte[] uncompressed = codec.Decompress(compressed, compressed.Length); + + Assert.IsTrue(Enumerable.SequenceEqual(data, uncompressed)); + } + } + + [Test] + public void ToStringAndName() + { + SnappyCodec codec = new SnappyCodec(); + + Assert.AreEqual("snappy", codec.GetName()); + Assert.AreEqual("snappy", codec.ToString()); + } + + [Test] + public void Equal() + { + SnappyCodec codec1 = new SnappyCodec(); + SnappyCodec codec2 = new SnappyCodec(); + + Assert.IsTrue(codec1.Equals(codec1)); + Assert.IsTrue(codec2.Equals(codec2)); + Assert.IsTrue(codec1.Equals(codec2)); + Assert.IsTrue(codec2.Equals(codec1)); + } + + [Test] + public void HashCode() + { + SnappyCodec codec = new SnappyCodec(); + + Assert.AreNotEqual(0, codec.GetHashCode()); + } + } +} diff --git a/lang/csharp/src/apache/codec/Avro.File.Snappy/Avro.File.Snappy.csproj b/lang/csharp/src/apache/codec/Avro.File.Snappy/Avro.File.Snappy.csproj new file mode 100644 index 00000000000..71bc7968827 --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.File.Snappy/Avro.File.Snappy.csproj @@ -0,0 +1,47 @@ + + + + + + + $(DefaultLibraryTargetFrameworks) + Avro.File.Snappy + true + ../../../../Avro.snk + + + + + Apache.Avro.File.Snappy + Snappy compression library for Apache.Avro + + + + true + + + + + + + + + + + + diff --git a/lang/csharp/src/apache/codec/Avro.File.Snappy/Crc32.cs b/lang/csharp/src/apache/codec/Avro.File.Snappy/Crc32.cs new file mode 100644 index 00000000000..1b832e11e43 --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.File.Snappy/Crc32.cs @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using System; + +namespace Avro.File.Snappy +{ + /// + /// Implements a 32-bit CRC hash algorithm. + /// + internal static class Crc32 + { + private const uint DefaultPolynomial = 0xedb88320u; + private const uint DefaultSeed = 0xffffffffu; + + private static uint[] defaultTable; + + public static uint Compute(byte[] buffer) + { + return Compute(DefaultPolynomial, DefaultSeed, buffer); + } + + public static uint Compute(uint polynomial, uint seed, ReadOnlySpan buffer) + { + return ~CalculateHash(InitializeTable(polynomial), seed, buffer); + } + + private static uint[] InitializeTable(uint polynomial) + { + if (polynomial == DefaultPolynomial && defaultTable != null) + return defaultTable; + + uint[] createTable = new uint[256]; + for (int i = 0; i < 256; i++) + { + uint entry = (uint)i; + for (int j = 0; j < 8; j++) + if ((entry & 1) == 1) + entry = (entry >> 1) ^ polynomial; + else + entry >>= 1; + createTable[i] = entry; + } + + if (polynomial == DefaultPolynomial) + defaultTable = createTable; + + return createTable; + } + + private static uint CalculateHash(uint[] table, uint seed, ReadOnlySpan buffer) + { + uint hash = seed; + for (int i = 0; i < buffer.Length; i++) + hash = (hash >> 8) ^ table[buffer[i] ^ hash & 0xff]; + return hash; + } + } +} diff --git a/lang/csharp/src/apache/codec/Avro.File.Snappy/Snappy.cs b/lang/csharp/src/apache/codec/Avro.File.Snappy/Snappy.cs new file mode 100644 index 00000000000..a73f8209c0d --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.File.Snappy/Snappy.cs @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using System; +using System.IO; + +namespace Avro.File.Snappy +{ + /// + /// Implements Snappy compression and decompression. + /// + public class SnappyCodec : Codec + { + /// + /// + public override byte[] Compress(byte[] uncompressedData) + { + using (MemoryStream outputStream = new MemoryStream()) + { + byte[] compressedData = IronSnappy.Snappy.Encode(uncompressedData); + outputStream.Write(compressedData, 0, compressedData.Length); + + var crc = ByteSwap(Crc32.Compute(uncompressedData)); + outputStream.Write(BitConverter.GetBytes(crc), 0, 4); + + return outputStream.ToArray(); + } + } + + /// + public override void Compress(MemoryStream inputStream, MemoryStream outputStream) + { + inputStream.Position = 0; + + byte[] uncompressedData = inputStream.ToArray(); + byte[] compressedData = IronSnappy.Snappy.Encode(uncompressedData); + + outputStream.SetLength(0); + + outputStream.Write(compressedData, 0, compressedData.Length); + + var crc = ByteSwap(Crc32.Compute(uncompressedData)); + outputStream.Write(BitConverter.GetBytes(crc), 0, 4); + } + + /// + public override byte[] Decompress(byte[] compressedData, int blockLength) + { + byte[] uncompressedData = IronSnappy.Snappy.Decode(compressedData.AsSpan(0, blockLength - 4)); + + return ByteSwap(Crc32.Compute(uncompressedData)) == BitConverter.ToUInt32(compressedData, blockLength - 4) ? + uncompressedData : + throw new IOException("Checksum failure"); + } + + private static uint ByteSwap(uint word) + { + return ((word >> 24) & 0x000000FF) | ((word >> 8) & 0x0000FF00) | ((word << 8) & 0x00FF0000) | ((word << 24) & 0xFF000000); + } + + /// + public override string GetName() + { + return DataFileConstants.SnappyCodec; + } + + /// + public override bool Equals(object other) + { + return this == other || GetType().Name == other.GetType().Name; + } + + /// + public override int GetHashCode() + { + return GetName().GetHashCode(); + } + } +} diff --git a/lang/csharp/src/apache/codec/Avro.File.XZ.Test/Avro.File.XZ.Test.csproj b/lang/csharp/src/apache/codec/Avro.File.XZ.Test/Avro.File.XZ.Test.csproj new file mode 100644 index 00000000000..354c6a51c26 --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.File.XZ.Test/Avro.File.XZ.Test.csproj @@ -0,0 +1,42 @@ + + + + + + + $(DefaultUnitTestTargetFrameworks) + false + + + + true + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/lang/csharp/src/apache/codec/Avro.File.XZ.Test/XZTests.cs b/lang/csharp/src/apache/codec/Avro.File.XZ.Test/XZTests.cs new file mode 100644 index 00000000000..27f38dc2bf2 --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.File.XZ.Test/XZTests.cs @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using System.IO; +using System.Linq; +using NUnit.Framework; + +namespace Avro.File.XZ.Test +{ + public class Tests + { + private static readonly int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 100000 }; + + [Test, Combinatorial] + public void CompressDecompress([ValueSource(nameof(_testLengths))] int length, [Values] XZLevel level) + { + byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); + + XZCodec codec = new XZCodec(level); + + byte[] compressed = codec.Compress(data); + byte[] uncompressed = codec.Decompress(compressed, compressed.Length); + + Assert.IsTrue(Enumerable.SequenceEqual(data, uncompressed)); + } + + [Test, Combinatorial] + public void CompressDecompressStream([ValueSource(nameof(_testLengths))] int length, [Values] XZLevel level) + { + byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); + + XZCodec codec = new XZCodec(level); + + using (MemoryStream inputStream = new MemoryStream(data)) + using (MemoryStream outputStream = new MemoryStream()) + { + codec.Compress(inputStream, outputStream); + + byte[] compressed = outputStream.ToArray(); + byte[] uncompressed = codec.Decompress(compressed, compressed.Length); + + Assert.IsTrue(Enumerable.SequenceEqual(data, uncompressed)); + } + } + + [Test] + public void ToStringAndName([Values] XZLevel level) + { + XZCodec codec = new XZCodec(level); + + Assert.AreEqual("xz", codec.GetName()); + Assert.AreEqual($"xz-{(int)level}", codec.ToString()); + } + + [Test] + public void DefaultLevel() + { + XZCodec codec = new XZCodec(); + + Assert.AreEqual(XZLevel.Default, codec.Level); + } + + [Test] + public void Equal([Values] XZLevel level) + { + XZCodec codec1 = new XZCodec(level); + XZCodec codec2 = new XZCodec(level); + + Assert.IsTrue(codec1.Equals(codec1)); + Assert.IsTrue(codec2.Equals(codec2)); + Assert.IsTrue(codec1.Equals(codec2)); + Assert.IsTrue(codec2.Equals(codec1)); + } + + [Test] + public void HashCode([Values] XZLevel level) + { + XZCodec codec = new XZCodec(level); + + Assert.AreNotEqual(0, codec.GetHashCode()); + } + } +} diff --git a/lang/csharp/src/apache/codec/Avro.File.XZ/Avro.File.XZ.csproj b/lang/csharp/src/apache/codec/Avro.File.XZ/Avro.File.XZ.csproj new file mode 100644 index 00000000000..034bb99dcce --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.File.XZ/Avro.File.XZ.csproj @@ -0,0 +1,48 @@ + + + + + + + $(DefaultLibraryTargetFrameworks) + Avro.File.XZ + true + ../../../../Avro.snk + CS8002 + + + + + Apache.Avro.File.XZ + XZ compression library for Apache.Avro + + + + true + + + + + + + + + + + + diff --git a/lang/csharp/src/apache/codec/Avro.File.XZ/XZ.cs b/lang/csharp/src/apache/codec/Avro.File.XZ/XZ.cs new file mode 100644 index 00000000000..84d3742f6cc --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.File.XZ/XZ.cs @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.InteropServices; +using Joveler.Compression.XZ; + +namespace Avro.File.XZ +{ + /// + /// XZ Compression level + /// + public enum XZLevel + { + Level0 = 0, + Level1 = 1, + Level2 = 2, + Level3 = 3, + Level4 = 4, + Level5 = 5, + Level6 = 6, + Level7 = 7, + Level8 = 8, + Level9 = 9, + Default = Level6, + Minimum = Level0, + Maximum = Level9 + } + + /// + /// Implements XZ compression and decompression. + /// + public class XZCodec : Codec + { + public XZLevel Level {get; private set;} + public bool Extreme {get; private set;} + public int Threads {get; private set;} + + public XZCodec() + : this(XZLevel.Default) + { + } + + public XZCodec(XZLevel level) + : this(level, false) + { + } + + public XZCodec(XZLevel level, bool extreme) + : this(level, extreme, 0) + { + } + + public XZCodec(XZLevel level, bool extreme, int numOfThreads) + { + Level = level; + Extreme = extreme; + Threads = numOfThreads; + } + + static XZCodec() + { + Initialize(); // One time initialization + } + + private static void Initialize() + { + string arch = RuntimeInformation.OSArchitecture.ToString().ToLower(); + string foundLibPath = string.Empty; + string libPath; + string rid; + string libName; + + // Determine Platform (needed for proper Runtime ID) + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + rid = $"win-{arch}"; + libName = "liblzma.dll"; + } + else + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + rid = $"linux-{arch}"; + libName = "liblzma.so"; + } + else + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + rid = $"osx-{arch}"; + libName = "liblzma.dylib"; + } + else + { + // Unknown platform + throw new PlatformNotSupportedException("Unknown runtime platform!"); + } + + // Try to search for the lib in the working directory and the application binary directory + foreach (var relPath in new List { ".", AppDomain.CurrentDomain.BaseDirectory }) + { + // Try first the lib name directly + libPath = Path.Combine(relPath, libName); + if (System.IO.File.Exists(libPath)) + { + foundLibPath = libPath; + break; + } + + // Try the runtimes/RID/native location + // This is the default location for netstandard native libs + libPath = Path.Combine(relPath, "runtimes", rid, "native", libName); + if (System.IO.File.Exists(libPath)) + { + foundLibPath = libPath; + break; + } + } + + // Try the OS search path if nothing is found yet + if (string.IsNullOrEmpty(foundLibPath)) + { + var values = Environment.GetEnvironmentVariable("PATH"); + foreach (string path in values.Split(Path.PathSeparator)) + { + libPath = Path.Combine(path, libName); + if (System.IO.File.Exists(libPath)) + { + foundLibPath = libPath; + break; + } + } + } + + if (string.IsNullOrEmpty(foundLibPath)) + throw new PlatformNotSupportedException($"Unable to find {libName}"); + + // Initialize XZ library + XZInit.GlobalInit(foundLibPath); + } + + public static void Uninitialize() + { + XZInit.GlobalCleanup(); + } + + /// + public override byte[] Compress(byte[] uncompressedData) + { + using (MemoryStream inputStream = new MemoryStream(uncompressedData)) + using (MemoryStream outputStream = new MemoryStream()) + { + Compress(inputStream, outputStream); + return outputStream.ToArray(); + } + } + + /// + public override void Compress(MemoryStream inputStream, MemoryStream outputStream) + { + XZCompressOptions compOpts = new XZCompressOptions + { + Level = (LzmaCompLevel)(int)Level, + ExtremeFlag = Extreme, + LeaveOpen = true + }; + + XZThreadedCompressOptions threadOpts = new XZThreadedCompressOptions + { + Threads = Threads, + }; + + inputStream.Position = 0; + outputStream.SetLength(0); + + using (XZStream xzStream = new XZStream(outputStream, compOpts, threadOpts)) + { + inputStream.CopyTo(xzStream); + xzStream.Flush(); + } + } + + /// + public override byte[] Decompress(byte[] compressedData, int blockLength) + { + XZDecompressOptions decompOpts = new XZDecompressOptions(); + + using (MemoryStream inputStream = new MemoryStream(compressedData, 0, blockLength)) + using (MemoryStream outputStream = new MemoryStream()) + using (XZStream xzStream = new XZStream(inputStream, decompOpts)) + { + xzStream.CopyTo(outputStream); + xzStream.Flush(); + return outputStream.ToArray(); + } + } + + /// + public override string GetName() + { + return DataFileConstants.XZCodec; + } + + /// + public override bool Equals(object other) + { + return this == other || GetType().Name == other.GetType().Name; + } + + /// + public override int GetHashCode() + { + return GetName().GetHashCode(); + } + + /// + public override string ToString() + { + return $"{GetName()}-{(int)Level}"; + } + } +} diff --git a/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/Avro.File.Zstandard.Test.csproj b/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/Avro.File.Zstandard.Test.csproj new file mode 100644 index 00000000000..651fabded68 --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/Avro.File.Zstandard.Test.csproj @@ -0,0 +1,42 @@ + + + + + + + $(DefaultUnitTestTargetFrameworks) + false + + + + true + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/ZstandardTests.cs b/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/ZstandardTests.cs new file mode 100644 index 00000000000..e360ee576c8 --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/ZstandardTests.cs @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using System.IO; +using System.Linq; +using NUnit.Framework; + +namespace Avro.File.Zstandard.Test +{ + public class Tests + { + private static readonly int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 100000 }; + + [Test, Combinatorial] + public void CompressDecompress([ValueSource(nameof(_testLengths))] int length, [Values] ZstandardLevel level) + { + byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); + + ZstandardCodec codec = new ZstandardCodec(level); + + byte[] compressed = codec.Compress(data); + byte[] uncompressed = codec.Decompress(compressed, compressed.Length); + + Assert.IsTrue(Enumerable.SequenceEqual(data, uncompressed)); + } + + [Test, Combinatorial] + public void CompressDecompressStream([ValueSource(nameof(_testLengths))] int length, [Values] ZstandardLevel level) + { + byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); + + ZstandardCodec codec = new ZstandardCodec(level); + + using (MemoryStream inputStream = new MemoryStream(data)) + using (MemoryStream outputStream = new MemoryStream()) + { + codec.Compress(inputStream, outputStream); + + byte[] compressed = outputStream.ToArray(); + byte[] uncompressed = codec.Decompress(compressed, compressed.Length); + + Assert.IsTrue(Enumerable.SequenceEqual(data, uncompressed)); + } + } + + [Test] + public void ToStringAndName([Values] ZstandardLevel level) + { + ZstandardCodec codec = new ZstandardCodec(level); + + Assert.AreEqual("zstandard", codec.GetName()); + Assert.AreEqual($"zstandard[{(int)level}]", codec.ToString()); + } + + [Test] + public void DefaultLevel() + { + ZstandardCodec codec = new ZstandardCodec(); + + Assert.AreEqual(ZstandardLevel.Default, codec.Level); + } + + [Test] + public void Equal([Values] ZstandardLevel level) + { + ZstandardCodec codec1 = new ZstandardCodec(level); + ZstandardCodec codec2 = new ZstandardCodec(level); + + Assert.IsTrue(codec1.Equals(codec1)); + Assert.IsTrue(codec2.Equals(codec2)); + Assert.IsTrue(codec1.Equals(codec2)); + Assert.IsTrue(codec2.Equals(codec1)); + } + + [Test] + public void HashCode([Values] ZstandardLevel level) + { + ZstandardCodec codec = new ZstandardCodec(level); + + Assert.AreNotEqual(0, codec.GetHashCode()); + } + } +} diff --git a/lang/csharp/src/apache/codec/Avro.File.Zstandard/Avro.File.Zstandard.csproj b/lang/csharp/src/apache/codec/Avro.File.Zstandard/Avro.File.Zstandard.csproj new file mode 100644 index 00000000000..17f9f9f00f5 --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.File.Zstandard/Avro.File.Zstandard.csproj @@ -0,0 +1,48 @@ + + + + + + + $(DefaultLibraryTargetFrameworks) + Avro.File.Zstandard + true + ../../../../Avro.snk + CS8002 + + + + + Apache.Avro.File.Zstandard + Zstandard compression library for Apache.Avro + + + + true + + + + + + + + + + + + diff --git a/lang/csharp/src/apache/codec/Avro.File.Zstandard/Zstandard.cs b/lang/csharp/src/apache/codec/Avro.File.Zstandard/Zstandard.cs new file mode 100644 index 00000000000..5adfb441799 --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.File.Zstandard/Zstandard.cs @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using System.IO; +using System.IO.Compression; +using Zstandard.Net; + +namespace Avro.File.Zstandard +{ + /// + /// Zstandard Compression level + /// + public enum ZstandardLevel + { + Level1 = 1, + Level2 = 2, + Level3 = 3, + Level4 = 4, + Level5 = 5, + Level6 = 6, + Level7 = 7, + Level8 = 8, + Level9 = 9, + Level10 = 10, + Level11 = 11, + Level12 = 12, + Level13 = 13, + Level14 = 14, + Level15 = 15, + Level16 = 16, + Level17 = 17, + Level18 = 18, + Level19 = 19, + Default = Level3, + Minimum = Level1, + Maximum = Level19 + } + + /// + /// Implements Zstandard compression and decompression. + /// + public class ZstandardCodec : Codec + { + public ZstandardLevel Level {get; private set;} + + public ZstandardCodec() + : this(ZstandardLevel.Default) + { + } + + public ZstandardCodec(ZstandardLevel level) + { + Level = level; + } + + /// + public override byte[] Compress(byte[] uncompressedData) + { + using (var outputStream = new MemoryStream()) + using (var compressionStream = new ZstandardStream(outputStream, CompressionMode.Compress)) + { + compressionStream.CompressionLevel = (int)Level; + compressionStream.Write(uncompressedData, 0, uncompressedData.Length); + compressionStream.Flush(); + return outputStream.ToArray(); + } + } + + /// + public override void Compress(MemoryStream inputStream, MemoryStream outputStream) + { + inputStream.Position = 0; + outputStream.SetLength(0); + + using (var compressionStream = new ZstandardStream(outputStream, CompressionMode.Compress, true)) + { + compressionStream.CompressionLevel = (int)Level; + inputStream.CopyTo(compressionStream); + compressionStream.Flush(); + } + } + + /// + public override byte[] Decompress(byte[] compressedData, int blockLength) + { + using (var memoryStream = new MemoryStream(compressedData, 0, blockLength)) + using (var outputStream = new MemoryStream()) + using (var compressionStream = new ZstandardStream(memoryStream, CompressionMode.Decompress)) + { + compressionStream.CopyTo(outputStream); + compressionStream.Flush(); + return outputStream.ToArray(); + } + } + + /// + public override string GetName() + { + return DataFileConstants.ZstandardCodec; + } + + /// + public override bool Equals(object other) + { + return this == other || GetType().Name == other.GetType().Name; + } + + /// + public override int GetHashCode() + { + return GetName().GetHashCode(); + } + + /// + public override string ToString() + { + return $"{GetName()}[{(int)Level}]"; + } + } +} diff --git a/lang/csharp/src/apache/main/File/Codec.cs b/lang/csharp/src/apache/main/File/Codec.cs index 3a5b27bea47..46191997a1d 100644 --- a/lang/csharp/src/apache/main/File/Codec.cs +++ b/lang/csharp/src/apache/main/File/Codec.cs @@ -19,6 +19,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Reflection; namespace Avro.File { @@ -101,6 +102,26 @@ public enum Type /// Codec that does not perform any compression. /// Null, + + /// + /// Codec type that implements the "Snappy" compression algorithm. + /// + Snappy, + + /// + /// Codec type that implements the "BZip2" compression algorithm. + /// + BZip2, + + /// + /// Codec type that implements the "XZ" compression algorithm. + /// + XZ, + + /// + /// Codec type that implements the "Zstandard" compression algorithm. + /// + Zstandard } /// @@ -140,6 +161,15 @@ public static Codec CreateCodec(Type codecType) return new DeflateCodec(); case Type.Null: return new NullCodec(); + case Type.Snappy: + case Type.BZip2: + case Type.XZ: + case Type.Zstandard: + { + // Create codec dynamically from "Avro.File.CODECNAME" assembly + Assembly assembly = Assembly.Load($"Avro.File.{codecType}"); + return assembly.CreateInstance($"Avro.File.{codecType}.{codecType}Codec") as Codec; + } } throw new AvroRuntimeException($"Unrecognized codec: {codecType}"); @@ -174,6 +204,14 @@ public static Codec CreateCodecFromString(string codecType) return CreateCodec(Type.Deflate); case DataFileConstants.NullCodec: return CreateCodec(Type.Null); + case DataFileConstants.SnappyCodec: + return CreateCodec(Type.Snappy); + case DataFileConstants.BZip2Codec: + return CreateCodec(Type.BZip2); + case DataFileConstants.XZCodec: + return CreateCodec(Type.XZ); + case DataFileConstants.ZstandardCodec: + return CreateCodec(Type.Zstandard); } throw new AvroRuntimeException($"Unrecognized codec: {codecType}"); diff --git a/lang/csharp/src/apache/main/File/DataFileConstants.cs b/lang/csharp/src/apache/main/File/DataFileConstants.cs index b99e0a4447f..438caeca0e4 100644 --- a/lang/csharp/src/apache/main/File/DataFileConstants.cs +++ b/lang/csharp/src/apache/main/File/DataFileConstants.cs @@ -48,6 +48,26 @@ public class DataFileConstants /// public const string DeflateCodec = "deflate"; + /// + /// Identifier for the Snappy codec. + /// + public const string SnappyCodec = "snappy"; + + /// + /// Identifier for the BZip2 codec. + /// + public const string BZip2Codec = "bzip2"; + + /// + /// Identifier for the XZ codec. + /// + public const string XZCodec = "xz"; + + /// + /// Identifier for the Zstandard codec. + /// + public const string ZstandardCodec = "zstandard"; + /// /// Reserved 'avro' metadata key. /// diff --git a/lang/csharp/src/apache/test/Avro.test.csproj b/lang/csharp/src/apache/test/Avro.test.csproj index a1864d89e04..6c359c81400 100644 --- a/lang/csharp/src/apache/test/Avro.test.csproj +++ b/lang/csharp/src/apache/test/Avro.test.csproj @@ -45,6 +45,13 @@ + + + + + + + diff --git a/lang/csharp/src/apache/test/File/FileTests.cs b/lang/csharp/src/apache/test/File/FileTests.cs index 70470434746..e54b353147a 100644 --- a/lang/csharp/src/apache/test/File/FileTests.cs +++ b/lang/csharp/src/apache/test/File/FileTests.cs @@ -34,32 +34,80 @@ public class FileTests const string specificSchema = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"Avro.Test.File\",\"fields\":" + "[{\"name\":\"name\",\"type\":[\"null\",\"string\"]},{\"name\":\"age\",\"type\":\"int\"}]}"; + private static IEnumerable TestSpecificDataSource() + { + foreach (Codec.Type codecType in Enum.GetValues(typeof(Codec.Type))) + { + yield return new TestCaseData(specificSchema, new object[] + { + new object[] { "John", 23 } + }, codecType).SetName("{m}(Case0,{2})"); + + yield return new TestCaseData(specificSchema, new object[] + { + new object[] { "John", 23 }, + new object[] { "Jane", 99 }, + new object[] { "Jeff", 88 } + }, codecType).SetName("{m}(Case1,{2})"); + + yield return new TestCaseData(specificSchema, new object[] + { + new object[] { "John", 23 }, + new object[] { "Jane", 99 }, + new object[] { "Jeff", 88 }, + new object[] { "James", 13 }, + new object[] { "June", 109 }, + new object[] { "Lloyd", 18 }, + new object[] {"Jenny", 3}, + new object[] { "Bob", 9 }, + new object[] { null, 48 } + }, codecType).SetName("{m}(Case2,{2})"); + + yield return new TestCaseData(specificSchema, new object[] + { + new object[] { "John", 23}, + new object[] { "Jane", 99 }, + new object[] { "Jeff", 88 }, + new object[] { "James", 13 }, + new object[] { "June", 109 }, + new object[] { "Lloyd", 18 }, + new object[] { "Jamie", 53 }, + new object[] { "Fanessa", 101 }, + new object[] { "Kan", 18 }, + new object[] { "Janey", 33 }, + new object[] { "Deva", 102 }, + new object[] { "Gavin", 28 }, + new object[] { "Lochy", 113 }, + new object[] { "Nickie", 10 }, + new object[] { "Liddia", 38 }, + new object[] { "Fred", 3 }, + new object[] { "April", 17 }, + new object[] { "Novac", 48 }, + new object[] { "Idan", 33 }, + new object[] { "Jolyon", 76 }, + new object[] { "Ant", 68 }, + new object[] { "Ernie", 43 }, + new object[] { "Joel", 99 }, + new object[] { "Dan", 78 }, + new object[] { "Dave", 103 }, + new object[] { "Hillary", 79 }, + new object[] { "Grant", 88 }, + new object[] { "JJ", 14 }, + new object[] { "Bill", 90 }, + new object[] { "Larry", 4 }, + new object[] { "Jenny", 3 }, + new object[] { "Bob", 9 }, + new object[] { null, 48 } + }, codecType).SetName("{m}(Case3,{2})"); + } + } + /// /// Reading & writing of specific (custom) record objects /// - /// /// /// - [TestCase(specificSchema, new object[] { new object[] { "John", 23 } }, Codec.Type.Deflate, TestName = "TestSpecificData0")] - [TestCase(specificSchema, new object[] { new object[] { "Jane", 23 } }, Codec.Type.Deflate, TestName = "TestSpecificData1")] - [TestCase(specificSchema, new object[] { new object[] { "John", 23 }, new object[] { "Jane", 99 }, new object[] { "Jeff", 88 } }, Codec.Type.Deflate, TestName = "TestSpecificData2")] - [TestCase(specificSchema, new object[] { new object[] {"John", 23}, new object[] { "Jane", 99 }, new object[] { "Jeff", 88 }, - new object[] {"James", 13}, new object[] { "June", 109 }, new object[] { "Lloyd", 18 }, - new object[] {"Jenny", 3}, new object[] { "Bob", 9 }, new object[] { null, 48 }}, Codec.Type.Deflate, TestName = "TestSpecificData3")] - [TestCase(specificSchema, new object[] { new object[] { "John", 23 } }, Codec.Type.Null, TestName = "TestSpecificData4")] - [TestCase(specificSchema, new object[] { new object[] { "Jane", 23 } }, Codec.Type.Null, TestName = "TestSpecificData5")] - [TestCase(specificSchema, new object[] { new object[] { "John", 23 }, new object[] { "Jane", 99 }, new object[] { "Jeff", 88 } }, Codec.Type.Null, TestName = "TestSpecificData6")] - [TestCase(specificSchema, new object[] { new object[] {"John", 23}, new object[] { "Jane", 99 }, new object[] { "Jeff", 88 }, - new object[] {"James", 13}, new object[] { "June", 109 }, new object[] { "Lloyd", 18 }, - new object[] {"Jamie", 53}, new object[] { "Fanessa", 101 }, new object[] { "Kan", 18 }, - new object[] {"Janey", 33}, new object[] { "Deva", 102 }, new object[] { "Gavin", 28 }, - new object[] {"Lochy", 113}, new object[] { "Nickie", 10 }, new object[] { "Liddia", 38 }, - new object[] {"Fred", 3}, new object[] { "April", 17 }, new object[] { "Novac", 48 }, - new object[] {"Idan", 33}, new object[] { "Jolyon", 76 }, new object[] { "Ant", 68 }, - new object[] {"Ernie", 43}, new object[] { "Joel", 99 }, new object[] { "Dan", 78 }, - new object[] {"Dave", 103}, new object[] { "Hillary", 79 }, new object[] { "Grant", 88 }, - new object[] {"JJ", 14}, new object[] { "Bill", 90 }, new object[] { "Larry", 4 }, - new object[] {"Jenny", 3}, new object[] { "Bob", 9 }, new object[] { null, 48 }}, Codec.Type.Null, TestName = "TestSpecificData7")] + [TestCaseSource(nameof(TestSpecificDataSource))] public void TestSpecificData(string schemaStr, object[] recs, Codec.Type codecType) { // create and write out @@ -95,6 +143,41 @@ public void TestSpecificData(string schemaStr, object[] recs, Codec.Type codecTy } } + private static IEnumerable TestAppendSpecificDataSource() + { + foreach (Codec.Type codecType in Enum.GetValues(typeof(Codec.Type))) + { + yield return new TestCaseData(specificSchema, + new object[] + { + new object[] { "John", 23 } + }, + new object[] + { + new object[] { "Jane", 21 } + }, codecType).SetName("{m}(Case0,{3})"); + + yield return new TestCaseData(specificSchema, + new object[] + { + new object[] { "John", 23 }, + new object[] { "Jane", 99 }, + new object[] { "Jeff", 88 }, + new object[] { "James", 13 }, + new object[] { "June", 109 }, + new object[] { "Lloyd", 18 }, + new object[] { "Jenny", 3 }, + new object[] { "Bob", 9 }, + new object[] { null, 48 } + }, + new object[] + { + new object[] { "Hillary", 79 }, + new object[] { "Grant", 88 } + }, codecType).SetName("{m}(Case1,{3})"); + } + } + /// /// Test appending of specific (custom) record objects /// @@ -102,18 +185,7 @@ public void TestSpecificData(string schemaStr, object[] recs, Codec.Type codecTy /// initial records /// append records /// initial compression codec type - [TestCase(specificSchema, new object[] { new object[] { "John", 23 } }, new object[] { new object[] { "Jane", 21 } }, Codec.Type.Deflate, TestName = "TestAppendSpecificData0")] - [TestCase(specificSchema, new object[] { new object[] { "John", 23 } }, new object[] { new object[] { "Jane", 21 } }, Codec.Type.Null, TestName = "TestAppendSpecificData1")] - [TestCase(specificSchema, new object[] { new object[] {"John", 23}, new object[] { "Jane", 99 }, new object[] { "Jeff", 88 }, - new object[] {"James", 13}, new object[] { "June", 109 }, new object[] { "Lloyd", 18 }, - new object[] {"Jenny", 3}, new object[] { "Bob", 9 }, new object[] { null, 48 }}, - new object[] { new object[] { "Hillary", 79 }, - new object[] { "Grant", 88 } }, Codec.Type.Deflate, TestName = "TestAppendSpecificData2")] - [TestCase(specificSchema, new object[] { new object[] {"John", 23}, new object[] { "Jane", 99 }, new object[] { "Jeff", 88 }, - new object[] {"James", 13}, new object[] { "June", 109 }, new object[] { "Lloyd", 18 }, - new object[] {"Jenny", 3}, new object[] { "Bob", 9 }, new object[] { null, 48 }}, - new object[] { new object[] { "Hillary", 79 }, - new object[] { "Grant", 88 } }, Codec.Type.Null, TestName = "TestAppendSpecificData3")] + [TestCaseSource(nameof(TestAppendSpecificDataSource))] public void TestAppendSpecificData(string schemaStr, object[] recs, object[] appendRecs, Codec.Type codecType) { IList records = MakeRecords(recs); @@ -161,84 +233,50 @@ public void TestAppendSpecificData(string schemaStr, object[] recs, object[] app } } + private static IEnumerable TestGenericDataSource() + { + foreach (Codec.Type codecType in Enum.GetValues(typeof(Codec.Type))) + { + yield return new TestCaseData( + "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"null\"}]}", new object[] { "f1", null }, codecType) + .SetName("{m}(null,{2})"); + yield return new TestCaseData( + "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"boolean\"}]}", new object[] { "f1", true }, codecType) + .SetName("{m}(true,{2})"); + yield return new TestCaseData( + "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"boolean\"}]}", new object[] { "f1", false }, codecType) + .SetName("{m}(false,{2})"); ; + yield return new TestCaseData( + "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"int\"}]}", new object[] { "f1", 101 }, codecType) + .SetName("{m}(int,{2})"); ; + yield return new TestCaseData( + "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"long\"}]}", new object[] { "f1", 101L }, codecType) + .SetName("{m}(long,{2})"); ; + yield return new TestCaseData( + "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"float\"}]}", new object[] { "f1", 101.78f }, codecType) + .SetName("{m}(float,{2})"); ; + yield return new TestCaseData( + "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"double\"}]}", new object[] { "f1", 101.78 }, codecType) + .SetName("{m}(double,{2})"); ; + yield return new TestCaseData( + "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"string\"}]}", new object[] { "f1", "A" }, codecType) + .SetName("{m}(string,{2})"); ; + yield return new TestCaseData( + "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"bytes\"}]}", new object[] { "f1", new byte[] { 0, 1 } }, codecType) + .SetName("{m}(bytes,{2})"); ; + } + } + /// /// Reading & writing of generic record objects /// /// /// /// - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"null\"}]}", - new object[] { "f1", null }, Codec.Type.Deflate)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"boolean\"}]}", - new object[] { "f1", true }, Codec.Type.Deflate)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"boolean\"}]}", - new object[] { "f1", false }, Codec.Type.Deflate)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"int\"}]}", - new object[] { "f1", 101 }, Codec.Type.Deflate)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"long\"}]}", - new object[] { "f1", 101L }, Codec.Type.Deflate)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"float\"}]}", - new object[] { "f1", 101.78f }, Codec.Type.Deflate)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"double\"}]}", - new object[] { "f1", 101.78 }, Codec.Type.Deflate)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"string\"}]}", - new object[] { "f1", "A" }, Codec.Type.Deflate)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"bytes\"}]}", - new object[] { "f1", new byte[] { 0, 1 } }, Codec.Type.Deflate)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" + - "[{\"name\":\"f1\", \"type\":{\"type\": \"enum\", \"name\": \"e\", \"symbols\":[\"s1\", \"s2\"]}}]}", - new object[] { "f1", "s2" }, Codec.Type.Deflate)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" + - "[{\"name\":\"f1\", \"type\":{\"type\": \"array\", \"items\": \"int\"}}]}", - new object[] { "f1", new object[] { 0, 1, 101 } }, Codec.Type.Deflate)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" + - "[{\"name\":\"f1\", \"type\":{\"type\": \"array\", \"items\": \"int\"}}]}", - new object[] { "f1", new int[] { 0, 1, 101 } }, Codec.Type.Deflate)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" + - "[{\"name\":\"f1\", \"type\":[\"int\", \"long\"]}]}", - new object[] { "f1", 100 }, Codec.Type.Deflate)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" + - "[{\"name\":\"f1\", \"type\":[\"int\", \"long\"]}]}", - new object[] { "f1", 100L }, Codec.Type.Deflate)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" + - "[{\"name\":\"f1\", \"type\":{\"type\": \"fixed\", \"name\": \"f\", \"size\": 2}}]}", - new object[] { "f1", new byte[] { 1, 2 } }, Codec.Type.Deflate)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"null\"}]}", - new object[] { "f1", null }, Codec.Type.Null)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"boolean\"}]}", - new object[] { "f1", true }, Codec.Type.Null)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"boolean\"}]}", - new object[] { "f1", false }, Codec.Type.Null)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"int\"}]}", - new object[] { "f1", 101 }, Codec.Type.Null)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"long\"}]}", - new object[] { "f1", 101L }, Codec.Type.Null)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"float\"}]}", - new object[] { "f1", 101.78f }, Codec.Type.Null)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"double\"}]}", - new object[] { "f1", 101.78 }, Codec.Type.Null)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"string\"}]}", - new object[] { "f1", "A" }, Codec.Type.Null)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"bytes\"}]}", - new object[] { "f1", new byte[] { 0, 1 } }, Codec.Type.Null)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" + - "[{\"name\":\"f1\", \"type\":{\"type\": \"enum\", \"name\": \"e\", \"symbols\":[\"s1\", \"s2\"]}}]}", - new object[] { "f1", "s2" }, Codec.Type.Null)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" + - "[{\"name\":\"f1\", \"type\":{\"type\": \"array\", \"items\": \"int\"}}]}", - new object[] { "f1", new object[] { 0, 1, 101 } }, Codec.Type.Null)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" + - "[{\"name\":\"f1\", \"type\":{\"type\": \"array\", \"items\": \"int\"}}]}", - new object[] { "f1", new int[] { 0, 1, 101 } }, Codec.Type.Null)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" + - "[{\"name\":\"f1\", \"type\":[\"int\", \"long\"]}]}", - new object[] { "f1", 100 }, Codec.Type.Null)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" + - "[{\"name\":\"f1\", \"type\":[\"int\", \"long\"]}]}", - new object[] { "f1", 100L }, Codec.Type.Null)] + [TestCaseSource(nameof(TestGenericDataSource))] public void TestGenericData(string schemaStr, object[] value, Codec.Type codecType) { - foreach(var rwFactory in GenericOptions()) + foreach (var rwFactory in GenericOptions()) { // Create and write out MemoryStream dataFileOutputStream = new MemoryStream(); @@ -264,6 +302,22 @@ public void TestGenericData(string schemaStr, object[] value, Codec.Type codecTy } } + private static IEnumerable TestAppendGenericDataSource() + { + foreach (Codec.Type codecType in Enum.GetValues(typeof(Codec.Type))) + { + yield return new TestCaseData( + "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"boolean\"}]}", new object[] { "f1", true }, new object[] { "f1", false }, codecType) + .SetName("{m}(bool,{3})"); + yield return new TestCaseData( + "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"int\"}]}", new object[] { "f1", 1 }, new object[] { "f1", 2 }, codecType) + .SetName("{m}(int,{3})"); + yield return new TestCaseData( + "{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"string\"}]}", new object[] { "f1", "A" }, new object[] { "f1", "B" }, codecType) + .SetName("{m}(string,{3})"); + } + } + /// /// Test appending of generic record objects /// @@ -271,10 +325,7 @@ public void TestGenericData(string schemaStr, object[] value, Codec.Type codecTy /// initial records /// append records /// innitial compression codec type - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"boolean\"}]}", - new object[] { "f1", true }, new object[] { "f1", false }, Codec.Type.Deflate)] - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"int\"}]}", - new object[] { "f1", 1 }, new object[] { "f1", 2 }, Codec.Type.Null)] + [TestCaseSource(nameof(TestAppendGenericDataSource))] public void TestAppendGenericData(string schemaStr, object[] recs, object[] appendRecs, Codec.Type codecType) { foreach (var rwFactory in GenericOptions()) @@ -346,9 +397,7 @@ public void OpenAppendWriter_IncorrectOutStream_Throws() /// DeflateStream as it is a standard non-seekable Stream that has the same behavior as the /// NetworkStream, which we should handle. /// - [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":" + - "[{\"name\":\"f1\", \"type\":[\"int\", \"long\"]}]}", - new object[] { "f1", 100L }, Codec.Type.Null)] + [TestCase("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":[\"int\", \"long\"]}]}", new object[] { "f1", 100L }, Codec.Type.Null)] public void TestNonSeekableStream(string schemaStr, object[] value, Codec.Type codecType) { foreach (var rwFactory in GenericOptions()) @@ -387,57 +436,41 @@ public void TestNonSeekableStream(string schemaStr, object[] value, Codec.Type c } } + private static IEnumerable TestPrimitiveDataSource() + { + foreach (Codec.Type codecType in Enum.GetValues(typeof(Codec.Type))) + { + yield return new TestCaseData("[\"boolean\", \"null\"]", null, codecType); + yield return new TestCaseData("[\"boolean\", \"null\"]", true, codecType); + yield return new TestCaseData("[\"int\", \"long\"]", 100, codecType); + yield return new TestCaseData("[\"int\", \"long\"]", 100L, codecType); + yield return new TestCaseData("[\"float\", \"double\"]", 100.75, codecType); + yield return new TestCaseData("[\"float\", \"double\"]", 23.67f, codecType); + yield return new TestCaseData("{\"type\": \"boolean\"}", true, codecType); + yield return new TestCaseData("{\"type\": \"boolean\"}", false, codecType); + yield return new TestCaseData("{\"type\": \"string\"}", "John", codecType); + yield return new TestCaseData("{\"type\": [\"null\",\"string\"]}", null, codecType); + yield return new TestCaseData("{\"type\": \"int\"}", 1, codecType); + yield return new TestCaseData("{\"type\": \"long\"}", 12312313123L, codecType); + yield return new TestCaseData("{\"type\": \"float\"}", 0.0f, codecType); + yield return new TestCaseData("{\"type\": \"double\"}", 0.0, codecType); + yield return new TestCaseData("[{\"type\": \"array\", \"items\": \"float\"}, \"double\"]", new float[] { 23.67f, 22.78f }, codecType); + yield return new TestCaseData("[{\"type\": \"array\", \"items\": \"float\"}, \"double\"]", 100.89, codecType); + yield return new TestCaseData("[{\"type\": \"array\", \"items\": \"string\"}, \"string\"]", "a", codecType); + yield return new TestCaseData("[{\"type\": \"array\", \"items\": \"string\"}, \"string\"]", new string[] { "a", "b" }, codecType); + yield return new TestCaseData("[{\"type\": \"array\", \"items\": \"bytes\"}, \"bytes\"]", new byte[] { 1, 2, 3 }, codecType); + yield return new TestCaseData("[{\"type\": \"array\", \"items\": \"bytes\"}, \"bytes\"]", new object[] { new byte[] { 1, 2 }, new byte[] { 3, 4 } }, codecType); + yield return new TestCaseData("[{\"type\": \"enum\", \"symbols\": [\"s1\", \"s2\"], \"name\": \"e\"}, \"string\"]", "h1", codecType); + } + } + /// /// Reading & writing of primitive objects /// /// /// /// - [TestCase("{\"type\": \"boolean\"}", true, Codec.Type.Deflate)] - [TestCase("{\"type\": \"boolean\"}", false, Codec.Type.Deflate)] - [TestCase("{\"type\": \"boolean\"}", true, Codec.Type.Null)] - [TestCase("{\"type\": \"boolean\"}", false, Codec.Type.Null)] - [TestCase("[\"boolean\", \"null\"]", null, Codec.Type.Deflate)] - [TestCase("[\"boolean\", \"null\"]", true, Codec.Type.Deflate)] - [TestCase("[\"int\", \"long\"]", 100, Codec.Type.Deflate)] - [TestCase("[\"int\", \"long\"]", 100L, Codec.Type.Deflate)] - [TestCase("[\"float\", \"double\"]", 100.75, Codec.Type.Deflate)] - [TestCase("[\"float\", \"double\"]", 23.67f, Codec.Type.Deflate)] - [TestCase("[{\"type\": \"array\", \"items\": \"float\"}, \"double\"]", new float[] { 23.67f, 22.78f }, Codec.Type.Deflate)] - [TestCase("[{\"type\": \"array\", \"items\": \"float\"}, \"double\"]", 100.89, Codec.Type.Deflate)] - [TestCase("[{\"type\": \"array\", \"items\": \"string\"}, \"string\"]", "a", Codec.Type.Deflate)] - [TestCase("[{\"type\": \"array\", \"items\": \"string\"}, \"string\"]", new string[] { "a", "b" }, Codec.Type.Deflate)] - [TestCase("[{\"type\": \"array\", \"items\": \"bytes\"}, \"bytes\"]", new byte[] { 1, 2, 3 }, Codec.Type.Deflate)] - [TestCase("[{\"type\": \"array\", \"items\": \"bytes\"}, \"bytes\"]", new object[] { new byte[] { 1, 2 }, new byte[] { 3, 4 } }, Codec.Type.Deflate)] - [TestCase("[{\"type\": \"enum\", \"symbols\": [\"s1\", \"s2\"], \"name\": \"e\"}, \"string\"]", "h1", Codec.Type.Deflate)] - [TestCase("{\"type\":\"string\"}", "John", Codec.Type.Deflate)] - [TestCase("{\"type\":[\"null\",\"string\"]}", null, Codec.Type.Deflate)] - [TestCase("{\"type\":\"int\"}", 1, Codec.Type.Deflate)] - [TestCase("{\"type\":\"boolean\"}", false, Codec.Type.Deflate)] - [TestCase("{\"type\":\"long\"}", 12312313123L, Codec.Type.Deflate)] - [TestCase("{\"type\":\"float\"}", 0.0f, Codec.Type.Deflate)] - [TestCase("{\"type\":\"double\"}", 0.0, Codec.Type.Deflate)] - [TestCase("[\"boolean\", \"null\"]", null, Codec.Type.Null)] - [TestCase("[\"boolean\", \"null\"]", true, Codec.Type.Null)] - [TestCase("[\"int\", \"long\"]", 100, Codec.Type.Null)] - [TestCase("[\"int\", \"long\"]", 100L, Codec.Type.Null)] - [TestCase("[\"float\", \"double\"]", 100.75, Codec.Type.Null)] - [TestCase("[\"float\", \"double\"]", 23.67f, Codec.Type.Null)] - [TestCase("[{\"type\": \"array\", \"items\": \"float\"}, \"double\"]", new float[] { 23.67f, 22.78f }, Codec.Type.Null)] - [TestCase("[{\"type\": \"array\", \"items\": \"float\"}, \"double\"]", 100.89, Codec.Type.Null)] - [TestCase("[{\"type\": \"array\", \"items\": \"string\"}, \"string\"]", "a", Codec.Type.Null)] - [TestCase("[{\"type\": \"array\", \"items\": \"string\"}, \"string\"]", new string[] { "a", "b" }, Codec.Type.Null)] - [TestCase("[{\"type\": \"array\", \"items\": \"bytes\"}, \"bytes\"]", new byte[] { 1, 2, 3 }, Codec.Type.Null)] - [TestCase("[{\"type\": \"array\", \"items\": \"bytes\"}, \"bytes\"]", new object[] { new byte[] { 1, 2 }, new byte[] { 3, 4 } }, Codec.Type.Null)] - [TestCase("[{\"type\": \"enum\", \"symbols\": [\"s1\", \"s2\"], \"name\": \"e\"}, \"string\"]", "h1", Codec.Type.Null)] - [TestCase("{\"type\":\"string\"}", "John", Codec.Type.Null)] - [TestCase("{\"type\":[\"null\",\"string\"]}", null, Codec.Type.Null)] - [TestCase("{\"type\":\"int\"}", 1, Codec.Type.Null)] - [TestCase("{\"type\":\"boolean\"}", false, Codec.Type.Null)] - [TestCase("{\"type\":\"long\"}", 12312313123L, Codec.Type.Null)] - [TestCase("{\"type\":\"float\"}", 0.0f, Codec.Type.Null)] - [TestCase("{\"type\":\"double\"}", 0.0, Codec.Type.Null)] - [TestCase("{\"type\":\"string\"}", "test", Codec.Type.Null)] + [TestCaseSource(nameof(TestPrimitiveDataSource))] public void TestPrimitiveData(string schemaStr, object value, Codec.Type codecType) { foreach(var rwFactory in GenericOptions()) @@ -455,32 +488,28 @@ public void TestPrimitiveData(string schemaStr, object value, Codec.Type codecTy } } + private static IEnumerable TestMetaDataSource() + { + foreach (Codec.Type codecType in Enum.GetValues(typeof(Codec.Type))) + { + foreach (bool useTypeGetter in new bool[] { true, false }) + { + yield return new TestCaseData("bytesTest", new byte[] { 1, 2, 3 }, codecType, useTypeGetter); + yield return new TestCaseData("stringTest", "testVal", codecType, useTypeGetter); + yield return new TestCaseData("longTest", 12312313123L, codecType, useTypeGetter); + yield return new TestCaseData("bytesTest", new byte[] { 1 }, codecType, useTypeGetter); + yield return new TestCaseData("longTest", -1211212L, codecType, useTypeGetter); + } + } + } + /// /// Reading & writing of header meta data /// - /// /// /// - [TestCase("bytesTest", new byte[] { 1, 2, 3 }, Codec.Type.Null, true)] - [TestCase("stringTest", "testVal", Codec.Type.Null, true)] - [TestCase("longTest", 12312313123L, Codec.Type.Null, true)] - [TestCase("bytesTest", new byte[] { 1 }, Codec.Type.Null, true)] - [TestCase("longTest", -1211212L, Codec.Type.Null, true)] - [TestCase("bytesTest", new byte[] { 1, 2, 3 }, Codec.Type.Deflate, true)] - [TestCase("stringTest", "testVal", Codec.Type.Deflate, true)] - [TestCase("longTest", 12312313123L, Codec.Type.Deflate, true)] - [TestCase("bytesTest", new byte[] { 1 }, Codec.Type.Deflate, true)] - [TestCase("longTest", -21211212L, Codec.Type.Deflate, true)] - [TestCase("bytesTest", new byte[] { 1, 2, 3 }, Codec.Type.Null, false)] - [TestCase("stringTest", "testVal", Codec.Type.Null, false)] - [TestCase("longTest", 12312313123L, Codec.Type.Null, false)] - [TestCase("bytesTest", new byte[] { 1 }, Codec.Type.Null, false)] - [TestCase("longTest", -1211212L, Codec.Type.Null, false)] - [TestCase("bytesTest", new byte[] { 1, 2, 3 }, Codec.Type.Deflate, false)] - [TestCase("stringTest", "testVal", Codec.Type.Deflate, false)] - [TestCase("longTest", 12312313123L, Codec.Type.Deflate, false)] - [TestCase("bytesTest", new byte[] { 1 }, Codec.Type.Deflate, false)] - [TestCase("longTest", -21211212L, Codec.Type.Deflate, false)] + /// + [TestCaseSource(nameof(TestMetaDataSource))] public void TestMetaData(string key, object value, Codec.Type codecType, bool useTypeGetter) { // create and write out @@ -507,6 +536,20 @@ public void TestMetaData(string key, object value, Codec.Type codecType, bool us } } + private static IEnumerable TestPartialReadSource() + { + foreach (Codec.Type codecType in Enum.GetValues(typeof(Codec.Type))) + { + yield return new TestCaseData(specificSchema, codecType, 0, 330).SetName("{m}({1},{2},{3})"); + yield return new TestCaseData(specificSchema, codecType, 1, 330).SetName("{m}({1},{2},{3})"); + yield return new TestCaseData(specificSchema, codecType, 135, 330).SetName("{m}({1},{2},{3})"); + yield return new TestCaseData(specificSchema, codecType, 194, 264).SetName("{m}({1},{2},{3})"); + } + + // This is only for Null codec + yield return new TestCaseData(specificSchema, Codec.Type.Null, 888, 165).SetName("{m}({1},{2},{3})"); + } + /// /// Partial reading of file / stream from /// position in stream @@ -514,12 +557,7 @@ public void TestMetaData(string key, object value, Codec.Type codecType, bool us /// /// /// - [TestCase(specificSchema, Codec.Type.Null, 1, 330)] // 330 - [TestCase(specificSchema, Codec.Type.Null, 135, 330)] // 330 - [TestCase(specificSchema, Codec.Type.Null, 194, 264)] // 264 - [TestCase(specificSchema, Codec.Type.Null, 235, 264)] // 264 - [TestCase(specificSchema, Codec.Type.Null, 888, 165)] // 165 - [TestCase(specificSchema, Codec.Type.Null, 0, 330)] // 330 + [TestCaseSource(nameof(TestPartialReadSource))] public void TestPartialRead(string schemaStr, Codec.Type codecType, int position, int expectedRecords) { // create and write out @@ -569,11 +607,9 @@ public void TestPartialRead(string schemaStr, Codec.Type codecType, int position /// Tests reading from sync boundaries. /// /// - /// /// - [TestCase(specificSchema, Codec.Type.Null)] - [TestCase(specificSchema, Codec.Type.Deflate)] - public void TestPartialReadAll(string schemaStr, Codec.Type codecType) + [Test] + public void TestPartialReadAll([Values(specificSchema)] string schemaStr, [Values] Codec.Type codecType) { // create and write out IList records = MakeRecords(GetTestFooObject()); @@ -636,14 +672,11 @@ public void TestPartialReadAll(string schemaStr, Codec.Type codecType) /// Test leaveOpen flag /// /// - /// /// - /// - [TestCase(specificSchema, Codec.Type.Null, true, false)] - [TestCase(specificSchema, Codec.Type.Null, true, true)] - [TestCase(specificSchema, Codec.Type.Null, false, false)] - [TestCase(specificSchema, Codec.Type.Null, false, true)] - public void TestLeaveOpen(string schemaStr, Codec.Type codecType, bool leaveWriteOpen, bool leaveReadOpen) + /// + /// + [Test] + public void TestLeaveOpen([Values(specificSchema)] string schemaStr, [Values] Codec.Type codecType, [Values] bool leaveWriteOpen, [Values] bool leaveReadOpen) { // create and write out IList records = MakeRecords(GetTestFooObject()); @@ -723,19 +756,23 @@ private static void AssertNumRecordsFromPosition( IFileReader reader, long Assert.AreEqual( expectedRecords, readRecords, "didn't read expected records from position " + position ); } + private static IEnumerable TestSyncAndSeekPositionsSource() + { + foreach (Codec.Type codecType in Enum.GetValues(typeof(Codec.Type))) + { + yield return new TestCaseData(specificSchema, codecType, 2, 0, 1).SetName("{m}({1},{2},{3},{4})"); + yield return new TestCaseData(specificSchema, codecType, 10, 1, 4).SetName("{m}({1},{2},{3},{4})"); + yield return new TestCaseData(specificSchema, codecType, 200, 111, 15).SetName("{m}({1},{2},{3},{4})"); + yield return new TestCaseData(specificSchema, codecType, 1000, 588, 998).SetName("{m}({1},{2},{3},{4})"); + } + } + /// /// Reading all sync positions and /// verifying them with subsequent seek /// positions /// - [TestCase(specificSchema, Codec.Type.Null, 2, 0, 1)] - [TestCase(specificSchema, Codec.Type.Null, 10, 1, 4)] - [TestCase(specificSchema, Codec.Type.Null, 200, 111, 15)] - [TestCase(specificSchema, Codec.Type.Null, 1000, 588, 998)] - [TestCase(specificSchema, Codec.Type.Deflate, 2, 0, 1)] - [TestCase(specificSchema, Codec.Type.Deflate, 10, 1, 4)] - [TestCase(specificSchema, Codec.Type.Deflate, 200, 111, 15)] - [TestCase(specificSchema, Codec.Type.Deflate, 1000, 588, 998)] + [TestCaseSource(nameof(TestSyncAndSeekPositionsSource))] public void TestSyncAndSeekPositions(string schemaStr, Codec.Type codecType, int iterations, int firstSyncPosition, int secondSyncPosition) { // create and write out @@ -819,6 +856,44 @@ public void TestDifferentReaderSchema() } } + /// + /// Reading & writing many specific record objects + /// + /// + /// + [Test] + public void TestLargeSpecificData([Values] Codec.Type codecType, [Values(0, 1000, 100000)] int numOfRecords) + { + foreach (var rwFactory in SpecificOptions()) + { + MemoryStream dataFileOutputStream = new MemoryStream(); + Schema schema = Schema.Parse(specificSchema); + using (IFileWriter dataFileWriter = rwFactory.CreateWriter(dataFileOutputStream, schema, Codec.CreateCodec(codecType))) + { + for (int index = 0; index < numOfRecords; index++) + { + dataFileWriter.Append(new Foo() { name = $"Name-{index}", age = index }); + } + } + + MemoryStream dataFileInputStream = new MemoryStream(dataFileOutputStream.ToArray()); + + // Read back and verify + using (IFileReader reader = rwFactory.CreateReader(dataFileInputStream, null)) + { + int index = 0; + foreach (Foo record in reader.NextEntries) + { + Assert.AreEqual($"Name-{index}", record.name); + Assert.AreEqual(index, record.age); + index++; + } + + Assert.AreEqual(numOfRecords, index); + } + } + } + /// /// Reading and writing using optional codecs /// @@ -827,6 +902,10 @@ public void TestDifferentReaderSchema() [TestCase("zstd", true)] [TestCase("deflate", false)] [TestCase("null", false)] + [TestCase("snappy", false)] + [TestCase("bzip2", false)] + [TestCase("xz", false)] + [TestCase("zstandard", false)] public void TestOptionalCodecs(string codecToUse, bool expectResolverProvidedCodec) { var resolverProvidedCodec = false; @@ -886,14 +965,17 @@ public void UnknownCodecFromType(Codec.Type codec) [TestCase("deflate")] [TestCase("null")] [TestCase(null)] // If codec is absent, it is assumed to be "null" + [TestCase("snappy")] + [TestCase("bzip2")] + [TestCase("xz")] + [TestCase("zstandard")] public void KnownCodecFromString(string codec) { Assert.NotNull(Codec.CreateCodecFromString(codec)); } - [TestCase(Codec.Type.Deflate)] - [TestCase(Codec.Type.Null)] - public void KnownCodecFromType(Codec.Type codec) + [Test] + public void KnownCodecFromType([Values] Codec.Type codec) { Assert.NotNull(Codec.CreateCodec(codec)); } diff --git a/lang/csharp/src/apache/test/Interop/InteropDataConstants.cs b/lang/csharp/src/apache/test/Interop/InteropDataConstants.cs index 94bfb408d18..170e28eb5fa 100644 --- a/lang/csharp/src/apache/test/Interop/InteropDataConstants.cs +++ b/lang/csharp/src/apache/test/Interop/InteropDataConstants.cs @@ -25,7 +25,11 @@ public class InteropDataConstants public static readonly HashSet SupportedCodecNames = new HashSet { DataFileConstants.NullCodec, - DataFileConstants.DeflateCodec + DataFileConstants.DeflateCodec, + DataFileConstants.SnappyCodec, + DataFileConstants.BZip2Codec, + DataFileConstants.XZCodec, + DataFileConstants.ZstandardCodec }; } } \ No newline at end of file diff --git a/lang/csharp/versions.props b/lang/csharp/versions.props index 2ec4d2e46ef..6b767c60dd7 100644 --- a/lang/csharp/versions.props +++ b/lang/csharp/versions.props @@ -31,6 +31,12 @@ 4.3.0 4.7.0 4.7.0 + + + 1.3.3 + 1.3.0 + 4.1.0 + 1.1.7