From 6eb35b504af2967084705fdc437c51b818279e67 Mon Sep 17 00:00:00 2001 From: Zoltan Csizmadia Date: Fri, 11 Feb 2022 19:38:03 -0600 Subject: [PATCH 01/28] Initial extra codec support --- lang/csharp/src/apache/codec/Avro.Codec.sln | 46 ++++++ .../codec/BZip2/Avro.Codec.BZip2.csproj | 47 ++++++ lang/csharp/src/apache/codec/BZip2/BZip2.cs | 114 ++++++++++++++ .../codec/Snappy/Avro.Codec.Snappy.csproj | 47 ++++++ lang/csharp/src/apache/codec/Snappy/Crc32.cs | 74 +++++++++ lang/csharp/src/apache/codec/Snappy/Snappy.cs | 96 ++++++++++++ .../src/apache/codec/XZ/Avro.Codec.XZ.csproj | 48 ++++++ lang/csharp/src/apache/codec/XZ/XZ.cs | 145 ++++++++++++++++++ .../Zstandard/Avro.Codec.Zstandard.csproj | 48 ++++++ .../src/apache/codec/Zstandard/Zstandard.cs | 126 +++++++++++++++ 10 files changed, 791 insertions(+) create mode 100644 lang/csharp/src/apache/codec/Avro.Codec.sln create mode 100644 lang/csharp/src/apache/codec/BZip2/Avro.Codec.BZip2.csproj create mode 100644 lang/csharp/src/apache/codec/BZip2/BZip2.cs create mode 100644 lang/csharp/src/apache/codec/Snappy/Avro.Codec.Snappy.csproj create mode 100644 lang/csharp/src/apache/codec/Snappy/Crc32.cs create mode 100644 lang/csharp/src/apache/codec/Snappy/Snappy.cs create mode 100644 lang/csharp/src/apache/codec/XZ/Avro.Codec.XZ.csproj create mode 100644 lang/csharp/src/apache/codec/XZ/XZ.cs create mode 100644 lang/csharp/src/apache/codec/Zstandard/Avro.Codec.Zstandard.csproj create mode 100644 lang/csharp/src/apache/codec/Zstandard/Zstandard.cs diff --git a/lang/csharp/src/apache/codec/Avro.Codec.sln b/lang/csharp/src/apache/codec/Avro.Codec.sln new file mode 100644 index 00000000000..f06c28a1316 --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.Codec.sln @@ -0,0 +1,46 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.30114.105 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Avro.Codec.BZip2", "BZip2\Avro.Codec.BZip2.csproj", "{40201607-AA97-44FB-8353-6CDFA803FBBB}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Avro.Codec.XZ", "XZ\Avro.Codec.XZ.csproj", "{06396DF0-D656-4133-8BDC-EBF194D13AE4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Avro.Codec.Snappy", "Snappy\Avro.Codec.Snappy.csproj", "{C3F05185-2D65-4A63-B16D-6BE20A254C1F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Avro.main", "..\main\Avro.main.csproj", "{E5B71984-FAEC-43E2-B2AC-4413D737C5CD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Avro.Codec.Zstandard", "Zstandard\Avro.Codec.Zstandard.csproj", "{21E6C150-200F-484C-949F-C818BBE4F276}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {40201607-AA97-44FB-8353-6CDFA803FBBB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {40201607-AA97-44FB-8353-6CDFA803FBBB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {40201607-AA97-44FB-8353-6CDFA803FBBB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {40201607-AA97-44FB-8353-6CDFA803FBBB}.Release|Any CPU.Build.0 = Release|Any CPU + {06396DF0-D656-4133-8BDC-EBF194D13AE4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {06396DF0-D656-4133-8BDC-EBF194D13AE4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {06396DF0-D656-4133-8BDC-EBF194D13AE4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {06396DF0-D656-4133-8BDC-EBF194D13AE4}.Release|Any CPU.Build.0 = Release|Any CPU + {C3F05185-2D65-4A63-B16D-6BE20A254C1F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C3F05185-2D65-4A63-B16D-6BE20A254C1F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C3F05185-2D65-4A63-B16D-6BE20A254C1F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C3F05185-2D65-4A63-B16D-6BE20A254C1F}.Release|Any CPU.Build.0 = Release|Any CPU + {E5B71984-FAEC-43E2-B2AC-4413D737C5CD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E5B71984-FAEC-43E2-B2AC-4413D737C5CD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E5B71984-FAEC-43E2-B2AC-4413D737C5CD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E5B71984-FAEC-43E2-B2AC-4413D737C5CD}.Release|Any CPU.Build.0 = Release|Any CPU + {21E6C150-200F-484C-949F-C818BBE4F276}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {21E6C150-200F-484C-949F-C818BBE4F276}.Debug|Any CPU.Build.0 = Debug|Any CPU + {21E6C150-200F-484C-949F-C818BBE4F276}.Release|Any CPU.ActiveCfg = Release|Any CPU + {21E6C150-200F-484C-949F-C818BBE4F276}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/lang/csharp/src/apache/codec/BZip2/Avro.Codec.BZip2.csproj b/lang/csharp/src/apache/codec/BZip2/Avro.Codec.BZip2.csproj new file mode 100644 index 00000000000..e9123a55c91 --- /dev/null +++ b/lang/csharp/src/apache/codec/BZip2/Avro.Codec.BZip2.csproj @@ -0,0 +1,47 @@ + + + + + + + $(DefaultLibraryTargetFrameworks) + Avro.Codec.BZip2 + true + ../../../../Avro.snk + + + + + Apache.Avro.Codec.BZip2 + + + + + true + + + + + + + + + + + + diff --git a/lang/csharp/src/apache/codec/BZip2/BZip2.cs b/lang/csharp/src/apache/codec/BZip2/BZip2.cs new file mode 100644 index 00000000000..0a74dad1e2d --- /dev/null +++ b/lang/csharp/src/apache/codec/BZip2/BZip2.cs @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.IO; + +namespace Avro.Codec.BZip2 +{ + /// + /// BZip2 Compression level + /// + public enum BZip2Level + { + Level1 = 1, + Level2 = 2, + Level3 = 3, + Level4 = 4, + Level5 = 5, + Level6 = 6, + Level7 = 7, + Level8 = 8, + Level9 = 9, + Default = Level9, + Minimum = Level1, + Maximum = Level9 + } + + /// + /// Implements BZip2 compression and decompression. + /// + public class BZip2Codec : Avro.File.Codec + { + public const string DataFileConstant = "bzip2"; + + private BZip2Level _level; + + public BZip2Codec() + : this(BZip2Level.Default) + { + } + + public BZip2Codec(BZip2Level level) + { + _level = level; + } + + /// + public override byte[] Compress(byte[] uncompressedData) + { + using (MemoryStream inputStream = new MemoryStream(uncompressedData)) + using (MemoryStream outputStream = new MemoryStream()) + { + ICSharpCode.SharpZipLib.BZip2.BZip2.Compress(inputStream, outputStream, false, (int)_level); + return outputStream.ToArray(); + } + } + + /// + public override void Compress(MemoryStream inputStream, MemoryStream outputStream) + { + ICSharpCode.SharpZipLib.BZip2.BZip2.Compress(inputStream, outputStream, false, (int)_level); + } + + /// + public override byte[] Decompress(byte[] compressedData, int blockLength) + { + using (MemoryStream inputStream = new MemoryStream(compressedData)) + using (MemoryStream outputStream = new MemoryStream()) + { + ICSharpCode.SharpZipLib.BZip2.BZip2.Decompress(inputStream, outputStream, false); + return outputStream.ToArray(); + } + } + + /// + public override string GetName() + { + return DataFileConstant; + } + + /// + public override bool Equals(object other) + { + return this == other || GetType().Name == other.GetType().Name; + } + + /// + public override int GetHashCode() + { + return 0; + } + + /// + public override string ToString() + { + return $"{GetName()}-{(int)_level}"; + } + } +} \ No newline at end of file diff --git a/lang/csharp/src/apache/codec/Snappy/Avro.Codec.Snappy.csproj b/lang/csharp/src/apache/codec/Snappy/Avro.Codec.Snappy.csproj new file mode 100644 index 00000000000..035f07f1ab7 --- /dev/null +++ b/lang/csharp/src/apache/codec/Snappy/Avro.Codec.Snappy.csproj @@ -0,0 +1,47 @@ + + + + + + + $(DefaultLibraryTargetFrameworks) + Avro.Codec.Snappy + true + ../../../../Avro.snk + + + + + Apache.Avro.Codec.Snappy + + + + + true + + + + + + + + + + + + diff --git a/lang/csharp/src/apache/codec/Snappy/Crc32.cs b/lang/csharp/src/apache/codec/Snappy/Crc32.cs new file mode 100644 index 00000000000..f450e8317c1 --- /dev/null +++ b/lang/csharp/src/apache/codec/Snappy/Crc32.cs @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; + +namespace Avro.Codec.Snappy +{ + /// + /// Implements a 32-bit CRC hash algorithm. + /// + internal static class Crc32 + { + const uint DefaultPolynomial = 0xedb88320u; + const uint DefaultSeed = 0xffffffffu; + + static uint[] defaultTable; + + public static uint Compute(byte[] buffer) + { + return Compute(DefaultPolynomial, DefaultSeed, buffer); + } + + public static uint Compute(uint polynomial, uint seed, ReadOnlySpan buffer) + { + return ~CalculateHash(InitializeTable(polynomial), seed, buffer); + } + + static uint[] InitializeTable(uint polynomial) + { + if (polynomial == DefaultPolynomial && defaultTable != null) + return defaultTable; + + uint[] createTable = new uint[256]; + for (int i = 0; i < 256; i++) + { + uint entry = (uint)i; + for (int j = 0; j < 8; j++) + if ((entry & 1) == 1) + entry = (entry >> 1) ^ polynomial; + else + entry >>= 1; + createTable[i] = entry; + } + + if (polynomial == DefaultPolynomial) + defaultTable = createTable; + + return createTable; + } + + static uint CalculateHash(uint[] table, uint seed, ReadOnlySpan buffer) + { + uint hash = seed; + for (int i = 0; i < buffer.Length; i++) + hash = (hash >> 8) ^ table[buffer[i] ^ hash & 0xff]; + return hash; + } + } +} diff --git a/lang/csharp/src/apache/codec/Snappy/Snappy.cs b/lang/csharp/src/apache/codec/Snappy/Snappy.cs new file mode 100644 index 00000000000..c2c8540fb60 --- /dev/null +++ b/lang/csharp/src/apache/codec/Snappy/Snappy.cs @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.IO; +using IronSnappy; + +namespace Avro.Codec.Snappy +{ + /// + /// Implements Snappy compression and decompression. + /// + public class SnappyCodec : Avro.File.Codec + { + public const string DataFileConstant = "snappy"; + + /// + /// + public override byte[] Compress(byte[] uncompressedData) + { + using (MemoryStream outputStream = new MemoryStream()) + { + byte[] compressedData = IronSnappy.Snappy.Encode(uncompressedData); + outputStream.Write(compressedData, 0, compressedData.Length); + + var crc = ByteSwap(Crc32.Compute(uncompressedData)); + outputStream.Write(BitConverter.GetBytes(crc), 0, 4); + + return outputStream.ToArray(); + } + } + + /// + public override void Compress(MemoryStream inputStream, MemoryStream outputStream) + { + byte[] uncompressedData = inputStream.GetBuffer(); + byte[] compressedData = IronSnappy.Snappy.Encode(uncompressedData); + outputStream.Write(compressedData, 0, compressedData.Length); + + var crc = ByteSwap(Crc32.Compute(uncompressedData)); + outputStream.Write(BitConverter.GetBytes(crc), 0, 4); + } + + /// + public override byte[] Decompress(byte[] compressedData, int blockLength) + { + byte[] uncompressedData = IronSnappy.Snappy.Decode(compressedData.AsSpan(0, blockLength - 4)); + + var crc = ByteSwap(Crc32.Compute(uncompressedData)); + if (crc != BitConverter.ToUInt32(compressedData, blockLength - 4)) + { + throw new IOException("Checksum failure"); + } + + return uncompressedData; + } + + private static uint ByteSwap(uint word) + { + return ((word >> 24) & 0x000000FF) | ((word >> 8) & 0x0000FF00) | ((word << 8) & 0x00FF0000) | ((word << 24) & 0xFF000000); + } + + /// + public override string GetName() + { + return DataFileConstant; + } + + /// + public override bool Equals(object other) + { + return this == other || GetType().Name == other.GetType().Name; + } + + /// + public override int GetHashCode() + { + return 0; + } + } +} \ No newline at end of file diff --git a/lang/csharp/src/apache/codec/XZ/Avro.Codec.XZ.csproj b/lang/csharp/src/apache/codec/XZ/Avro.Codec.XZ.csproj new file mode 100644 index 00000000000..d38a4a989ca --- /dev/null +++ b/lang/csharp/src/apache/codec/XZ/Avro.Codec.XZ.csproj @@ -0,0 +1,48 @@ + + + + + + + $(DefaultLibraryTargetFrameworks) + Avro.Codec.XZ + true + ../../../../Avro.snk + CS8002 + + + + + Apache.Avro.Codec.XZ + + + + + true + + + + + + + + + + + + diff --git a/lang/csharp/src/apache/codec/XZ/XZ.cs b/lang/csharp/src/apache/codec/XZ/XZ.cs new file mode 100644 index 00000000000..e8359a5a905 --- /dev/null +++ b/lang/csharp/src/apache/codec/XZ/XZ.cs @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.IO; +using Joveler.Compression.XZ; + +namespace Avro.Codec.XZ +{ + /// + /// XZ Compression level + /// + public enum XZLevel + { + Level0 = 0, + Level1 = 1, + Level2 = 2, + Level3 = 3, + Level4 = 4, + Level5 = 5, + Level6 = 6, + Level7 = 7, + Level8 = 8, + Level9 = 9, + Default = Level6, + Minimum = Level0, + Maximum = Level9 + } + + /// + /// Implements XZ compression and decompression. + /// + public class XZCodec : Avro.File.Codec + { + public const string DataFileConstant = "xz"; + + private XZLevel _level; + private bool _extreme; + private int _threads; + + public XZCodec() + : this(XZLevel.Default, false) + { + } + + public XZCodec(XZLevel level, bool extreme) + : this(level, extreme, 0) + { + } + + public XZCodec(XZLevel level, bool extreme, int numOfThreads) + { + _level = level; + _extreme = extreme; + _threads = numOfThreads; + } + + /// + public override byte[] Compress(byte[] uncompressedData) + { + using (MemoryStream inputStream = new MemoryStream(uncompressedData)) + using (MemoryStream outputStream = new MemoryStream()) + { + Compress(inputStream, outputStream); + return outputStream.ToArray(); + } + } + + /// + public override void Compress(MemoryStream inputStream, MemoryStream outputStream) + { + XZCompressOptions compOpts = new XZCompressOptions + { + Level = (LzmaCompLevel)(int)_level, + ExtremeFlag = _extreme, + LeaveOpen = true + }; + + XZThreadedCompressOptions threadOpts = new XZThreadedCompressOptions + { + Threads = _threads, + }; + + using (XZStream xzStream = new XZStream(outputStream, compOpts, threadOpts)) + { + inputStream.CopyTo(xzStream); + xzStream.Flush(); + } + } + + /// + public override byte[] Decompress(byte[] compressedData, int blockLength) + { + XZDecompressOptions decompOpts = new XZDecompressOptions(); + + using (MemoryStream inputStream = new MemoryStream(compressedData)) + using (MemoryStream outputStream = new MemoryStream()) + using (XZStream xzStream = new XZStream(inputStream, decompOpts)) + { + xzStream.CopyTo(outputStream); + xzStream.Flush(); + return outputStream.ToArray(); + } + } + + /// + public override string GetName() + { + return DataFileConstant; + } + + /// + public override bool Equals(object other) + { + return this == other || GetType().Name == other.GetType().Name; + } + + /// + public override int GetHashCode() + { + return 0; + } + + /// + public override string ToString() + { + return $"{GetName()}-{(int)_level}"; + } + } +} \ No newline at end of file diff --git a/lang/csharp/src/apache/codec/Zstandard/Avro.Codec.Zstandard.csproj b/lang/csharp/src/apache/codec/Zstandard/Avro.Codec.Zstandard.csproj new file mode 100644 index 00000000000..4f912476620 --- /dev/null +++ b/lang/csharp/src/apache/codec/Zstandard/Avro.Codec.Zstandard.csproj @@ -0,0 +1,48 @@ + + + + + + + $(DefaultLibraryTargetFrameworks) + Avro.Codec.Zstandard + true + ../../../../Avro.snk + CS8002 + + + + + Apache.Avro.Codec.Zstandard + + + + + true + + + + + + + + + + + + diff --git a/lang/csharp/src/apache/codec/Zstandard/Zstandard.cs b/lang/csharp/src/apache/codec/Zstandard/Zstandard.cs new file mode 100644 index 00000000000..6978987ce73 --- /dev/null +++ b/lang/csharp/src/apache/codec/Zstandard/Zstandard.cs @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.IO; +using ZstdNet; + +namespace Avro.Codec.Zstandard +{ + /// + /// Zstandard Compression level + /// + public enum ZstandardLevel + { + Level1 = 1, + Level2 = 2, + Level3 = 3, + Level4 = 4, + Level5 = 5, + Level6 = 6, + Level7 = 7, + Level8 = 8, + Level9 = 9, + Level10 = 10, + Level11 = 11, + Level12 = 12, + Level13 = 13, + Level14 = 14, + Level15 = 15, + Level16 = 16, + Level17 = 17, + Level18 = 18, + Level19 = 19, + Default = Level3, + Minimum = Level1, + Maximum = Level19 + } + + /// + /// Implements BZip2 compression and decompression. + /// + public class ZstandardCodec : Avro.File.Codec + { + public const string DataFileConstant = "zstandard"; + + private ZstandardLevel _level; + + public ZstandardCodec() + : this(ZstandardLevel.Default) + { + } + + public ZstandardCodec(ZstandardLevel level) + { + _level = level; + } + + /// + public override byte[] Compress(byte[] uncompressedData) + { + using(CompressionOptions options = new CompressionOptions((int)_level)) + using(Compressor compressor = new Compressor(options)) + { + return compressor.Wrap(uncompressedData); + } + } + + /// + public override void Compress(MemoryStream inputStream, MemoryStream outputStream) + { + using(CompressionOptions options = new CompressionOptions((int)_level)) + using(CompressionStream compressionStream = new CompressionStream(outputStream, options)) + { + inputStream.CopyTo(compressionStream); + } + } + + /// + public override byte[] Decompress(byte[] compressedData, int blockLength) + { + using(Decompressor decompressor = new Decompressor()) + { + return decompressor.Unwrap(compressedData.AsSpan(0, blockLength)); + } + } + + /// + public override string GetName() + { + return DataFileConstant; + } + + /// + public override bool Equals(object other) + { + return this == other || GetType().Name == other.GetType().Name; + } + + /// + public override int GetHashCode() + { + return 0; + } + + /// + public override string ToString() + { + return $"{GetName()}[{(int)_level}]"; + } + } +} \ No newline at end of file From c04cd79dba744364e84eb4078ddb0f77355901d7 Mon Sep 17 00:00:00 2001 From: Zoltan Csizmadia Date: Sat, 12 Feb 2022 09:32:39 -0600 Subject: [PATCH 02/28] Add tests --- .../Avro.Codec.BZip2.Test.csproj | 42 +++++++ .../codec/Avro.Codec.BZip2.Test/BZip2Tests.cs | 37 +++++++ .../Avro.Codec.BZip2.csproj | 0 .../{BZip2 => Avro.Codec.BZip2}/BZip2.cs | 8 +- .../Avro.Codec.Snappy.Test.csproj | 42 +++++++ .../Avro.Codec.Snappy.Test/SnappyTests.cs | 33 ++++++ .../Avro.Codec.Snappy.csproj | 0 .../{Snappy => Avro.Codec.Snappy}/Crc32.cs | 0 .../{Snappy => Avro.Codec.Snappy}/Snappy.cs | 2 +- .../Avro.Codec.XZ.Test.csproj | 42 +++++++ .../codec/Avro.Codec.XZ.Test/XZTests.cs | 49 +++++++++ .../Avro.Codec.XZ.csproj | 0 .../apache/codec/{XZ => Avro.Codec.XZ}/XZ.cs | 103 +++++++++++++++++- .../Avro.Codec.Zstandard.Test.csproj | 42 +++++++ .../ZstandardTests.cs | 37 +++++++ .../Avro.Codec.Zstandard.csproj | 0 .../Zstandard.cs | 0 lang/csharp/src/apache/codec/Avro.Codec.sln | 47 ++++++-- 18 files changed, 466 insertions(+), 18 deletions(-) create mode 100644 lang/csharp/src/apache/codec/Avro.Codec.BZip2.Test/Avro.Codec.BZip2.Test.csproj create mode 100644 lang/csharp/src/apache/codec/Avro.Codec.BZip2.Test/BZip2Tests.cs rename lang/csharp/src/apache/codec/{BZip2 => Avro.Codec.BZip2}/Avro.Codec.BZip2.csproj (100%) rename lang/csharp/src/apache/codec/{BZip2 => Avro.Codec.BZip2}/BZip2.cs (97%) create mode 100644 lang/csharp/src/apache/codec/Avro.Codec.Snappy.Test/Avro.Codec.Snappy.Test.csproj create mode 100644 lang/csharp/src/apache/codec/Avro.Codec.Snappy.Test/SnappyTests.cs rename lang/csharp/src/apache/codec/{Snappy => Avro.Codec.Snappy}/Avro.Codec.Snappy.csproj (100%) rename lang/csharp/src/apache/codec/{Snappy => Avro.Codec.Snappy}/Crc32.cs (100%) rename lang/csharp/src/apache/codec/{Snappy => Avro.Codec.Snappy}/Snappy.cs (99%) create mode 100644 lang/csharp/src/apache/codec/Avro.Codec.XZ.Test/Avro.Codec.XZ.Test.csproj create mode 100644 lang/csharp/src/apache/codec/Avro.Codec.XZ.Test/XZTests.cs rename lang/csharp/src/apache/codec/{XZ => Avro.Codec.XZ}/Avro.Codec.XZ.csproj (100%) rename lang/csharp/src/apache/codec/{XZ => Avro.Codec.XZ}/XZ.cs (54%) create mode 100644 lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/Avro.Codec.Zstandard.Test.csproj create mode 100644 lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/ZstandardTests.cs rename lang/csharp/src/apache/codec/{Zstandard => Avro.Codec.Zstandard}/Avro.Codec.Zstandard.csproj (100%) rename lang/csharp/src/apache/codec/{Zstandard => Avro.Codec.Zstandard}/Zstandard.cs (100%) diff --git a/lang/csharp/src/apache/codec/Avro.Codec.BZip2.Test/Avro.Codec.BZip2.Test.csproj b/lang/csharp/src/apache/codec/Avro.Codec.BZip2.Test/Avro.Codec.BZip2.Test.csproj new file mode 100644 index 00000000000..22d3b5d3d8a --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.Codec.BZip2.Test/Avro.Codec.BZip2.Test.csproj @@ -0,0 +1,42 @@ + + + + + + + $(DefaultUnitTestTargetFrameworks) + false + + + + true + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/lang/csharp/src/apache/codec/Avro.Codec.BZip2.Test/BZip2Tests.cs b/lang/csharp/src/apache/codec/Avro.Codec.BZip2.Test/BZip2Tests.cs new file mode 100644 index 00000000000..0eb7b930295 --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.Codec.BZip2.Test/BZip2Tests.cs @@ -0,0 +1,37 @@ +using System.Linq; +using NUnit.Framework; + +namespace Avro.Codec.BZip2.Test +{ + public class Tests + { + [TestCase(0)] + [TestCase(1000)] + [TestCase(64 * 1024)] + [TestCase(1 * 1024 * 1024)] + public void CompressDecompress(int length) + { + byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); + + BZip2Codec codec = new BZip2Codec(); + + byte[] compressed = codec.Compress(data); + byte[] uncompressed = codec.Decompress(compressed, compressed.Length); + + CollectionAssert.AreEqual(data, uncompressed); + } + + [Test] + [TestCase(BZip2Level.Level1, ExpectedResult = "bzip2-1")] + [TestCase(BZip2Level.Level2, ExpectedResult = "bzip2-2")] + [TestCase(BZip2Level.Level3, ExpectedResult = "bzip2-3")] + public string ToStringAndName(BZip2Level level) + { + BZip2Codec codec = new BZip2Codec(level); + + Assert.AreEqual("bzip2", codec.GetName()); + + return codec.ToString(); + } + } +} diff --git a/lang/csharp/src/apache/codec/BZip2/Avro.Codec.BZip2.csproj b/lang/csharp/src/apache/codec/Avro.Codec.BZip2/Avro.Codec.BZip2.csproj similarity index 100% rename from lang/csharp/src/apache/codec/BZip2/Avro.Codec.BZip2.csproj rename to lang/csharp/src/apache/codec/Avro.Codec.BZip2/Avro.Codec.BZip2.csproj diff --git a/lang/csharp/src/apache/codec/BZip2/BZip2.cs b/lang/csharp/src/apache/codec/Avro.Codec.BZip2/BZip2.cs similarity index 97% rename from lang/csharp/src/apache/codec/BZip2/BZip2.cs rename to lang/csharp/src/apache/codec/Avro.Codec.BZip2/BZip2.cs index 0a74dad1e2d..3f2161e1fec 100644 --- a/lang/csharp/src/apache/codec/BZip2/BZip2.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.BZip2/BZip2.cs @@ -26,6 +26,7 @@ namespace Avro.Codec.BZip2 /// public enum BZip2Level { + Default = 9, Level1 = 1, Level2 = 2, Level3 = 3, @@ -34,10 +35,7 @@ public enum BZip2Level Level6 = 6, Level7 = 7, Level8 = 8, - Level9 = 9, - Default = Level9, - Minimum = Level1, - Maximum = Level9 + Level9 = 9 } /// @@ -111,4 +109,4 @@ public override string ToString() return $"{GetName()}-{(int)_level}"; } } -} \ No newline at end of file +} diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Snappy.Test/Avro.Codec.Snappy.Test.csproj b/lang/csharp/src/apache/codec/Avro.Codec.Snappy.Test/Avro.Codec.Snappy.Test.csproj new file mode 100644 index 00000000000..2c399618015 --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.Codec.Snappy.Test/Avro.Codec.Snappy.Test.csproj @@ -0,0 +1,42 @@ + + + + + + + $(DefaultUnitTestTargetFrameworks) + false + + + + true + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Snappy.Test/SnappyTests.cs b/lang/csharp/src/apache/codec/Avro.Codec.Snappy.Test/SnappyTests.cs new file mode 100644 index 00000000000..7c35c27b43f --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.Codec.Snappy.Test/SnappyTests.cs @@ -0,0 +1,33 @@ +using System.Linq; +using NUnit.Framework; + +namespace Avro.Codec.Snappy.Test +{ + public class Tests + { + [TestCase(0)] + [TestCase(1000)] + [TestCase(64*1024)] + [TestCase(1*1024*1024)] + public void CompressDecompress(int length) + { + byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); + + SnappyCodec codec = new SnappyCodec(); + + byte[] compressed = codec.Compress(data); + byte[] uncompressed = codec.Decompress(compressed, compressed.Length); + + CollectionAssert.AreEqual(data, uncompressed); + } + + [Test] + public void ToStringAndName() + { + SnappyCodec codec = new SnappyCodec(); + + Assert.AreEqual("snappy", codec.GetName()); + Assert.AreEqual("snappy", codec.ToString()); + } + } +} diff --git a/lang/csharp/src/apache/codec/Snappy/Avro.Codec.Snappy.csproj b/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Avro.Codec.Snappy.csproj similarity index 100% rename from lang/csharp/src/apache/codec/Snappy/Avro.Codec.Snappy.csproj rename to lang/csharp/src/apache/codec/Avro.Codec.Snappy/Avro.Codec.Snappy.csproj diff --git a/lang/csharp/src/apache/codec/Snappy/Crc32.cs b/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Crc32.cs similarity index 100% rename from lang/csharp/src/apache/codec/Snappy/Crc32.cs rename to lang/csharp/src/apache/codec/Avro.Codec.Snappy/Crc32.cs diff --git a/lang/csharp/src/apache/codec/Snappy/Snappy.cs b/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Snappy.cs similarity index 99% rename from lang/csharp/src/apache/codec/Snappy/Snappy.cs rename to lang/csharp/src/apache/codec/Avro.Codec.Snappy/Snappy.cs index c2c8540fb60..522440f7bd4 100644 --- a/lang/csharp/src/apache/codec/Snappy/Snappy.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Snappy.cs @@ -93,4 +93,4 @@ public override int GetHashCode() return 0; } } -} \ No newline at end of file +} diff --git a/lang/csharp/src/apache/codec/Avro.Codec.XZ.Test/Avro.Codec.XZ.Test.csproj b/lang/csharp/src/apache/codec/Avro.Codec.XZ.Test/Avro.Codec.XZ.Test.csproj new file mode 100644 index 00000000000..f400d1170f2 --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.Codec.XZ.Test/Avro.Codec.XZ.Test.csproj @@ -0,0 +1,42 @@ + + + + + + + $(DefaultUnitTestTargetFrameworks) + false + + + + true + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/lang/csharp/src/apache/codec/Avro.Codec.XZ.Test/XZTests.cs b/lang/csharp/src/apache/codec/Avro.Codec.XZ.Test/XZTests.cs new file mode 100644 index 00000000000..7108c421b2b --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.Codec.XZ.Test/XZTests.cs @@ -0,0 +1,49 @@ +using System.Linq; +using NUnit.Framework; + +namespace Avro.Codec.XZ.Test +{ + public class Tests + { + [OneTimeSetUp] + public void OneTimeSetup() + { + XZCodec.Initialize(); + } + + [OneTimeTearDown] + public void OneTimeTearDOwn() + { + XZCodec.Uninitialize(); + } + + [TestCase(0)] + [TestCase(1000)] + [TestCase(64 * 1024)] + [TestCase(1 * 1024 * 1024)] + public void CompressDecompress(int length) + { + byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); + + XZCodec codec = new XZCodec(); + + byte[] compressed = codec.Compress(data); + byte[] uncompressed = codec.Decompress(compressed, compressed.Length); + + CollectionAssert.AreEqual(data, uncompressed); + } + + [Test] + [TestCase(XZLevel.Level1, ExpectedResult = "xz-1")] + [TestCase(XZLevel.Level2, ExpectedResult = "xz-2")] + [TestCase(XZLevel.Level3, ExpectedResult = "xz-3")] + public string ToStringAndName(XZLevel level) + { + XZCodec codec = new XZCodec(level); + + Assert.AreEqual("xz", codec.GetName()); + + return codec.ToString(); + } + } +} diff --git a/lang/csharp/src/apache/codec/XZ/Avro.Codec.XZ.csproj b/lang/csharp/src/apache/codec/Avro.Codec.XZ/Avro.Codec.XZ.csproj similarity index 100% rename from lang/csharp/src/apache/codec/XZ/Avro.Codec.XZ.csproj rename to lang/csharp/src/apache/codec/Avro.Codec.XZ/Avro.Codec.XZ.csproj diff --git a/lang/csharp/src/apache/codec/XZ/XZ.cs b/lang/csharp/src/apache/codec/Avro.Codec.XZ/XZ.cs similarity index 54% rename from lang/csharp/src/apache/codec/XZ/XZ.cs rename to lang/csharp/src/apache/codec/Avro.Codec.XZ/XZ.cs index e8359a5a905..f3f4d4dbfcc 100644 --- a/lang/csharp/src/apache/codec/XZ/XZ.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.XZ/XZ.cs @@ -17,7 +17,9 @@ */ using System; +using System.Collections.Generic; using System.IO; +using System.Runtime.InteropServices; using Joveler.Compression.XZ; namespace Avro.Codec.XZ @@ -54,7 +56,12 @@ public class XZCodec : Avro.File.Codec private int _threads; public XZCodec() - : this(XZLevel.Default, false) + : this(XZLevel.Default) + { + } + + public XZCodec(XZLevel level) + : this(level, false) { } @@ -70,6 +77,98 @@ public XZCodec(XZLevel level, bool extreme, int numOfThreads) _threads = numOfThreads; } + // !!! + // !!! This must be called as a global init before using this library + // !!! TODO: Make it more convinient + // !!! + public static void Initialize() + { + string libPath; + string foundLibPath = string.Empty; + string libName = "liblzma"; + string rid; + string arch; + + switch(RuntimeInformation.OSArchitecture) + { + case Architecture.X86: + arch = "x86"; + break; + case Architecture.X64: + arch = "x64"; + break; + case Architecture.Arm: + arch = "arm"; + break; + case Architecture.Arm64: + arch = "arm64"; + break; + default: + throw new Exception("Unknown runtime architecture!"); + }; + + // Unknown architecture + if (arch == null) + throw new Exception("Unknown runtime architecture!"); + + // Determine Platform (needed for proper Runtime ID) + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + rid = $"win-{arch}"; + else + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + rid = $"linux-{arch}"; + else + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + rid = $"osx-{arch}"; + else + // Unknown platform + throw new Exception("Unknown runtime platform!"); + + // Determine Platform (needed for proper Runtime ID) + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + libName += ".dll"; + else + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + libName += ".so"; + else + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + libName += ".dylib"; + else + throw new PlatformNotSupportedException(); + + // Try to search for the lib in the working directory and the application binary directory + foreach (var relPath in new List { ".", AppDomain.CurrentDomain.BaseDirectory }) + { + // Try first the lib name directly + libPath = Path.Combine(relPath, libName); + if (System.IO.File.Exists(libPath)) + { + foundLibPath = libPath; + break; + } + + // Try the runtimes/RID/native location + // This is the default location for netstandard native libs + libPath = Path.Combine(relPath, "runtimes", rid, "native", libName); + if (System.IO.File.Exists(libPath)) + { + foundLibPath = libPath; + break; + } + } + + if (string.IsNullOrEmpty(foundLibPath)) + throw new PlatformNotSupportedException(); + + // Initialize XZ library + XZInit.GlobalInit(foundLibPath); + } + + public static void Uninitialize() + { + XZInit.GlobalCleanup(); + } + /// public override byte[] Compress(byte[] uncompressedData) { @@ -142,4 +241,4 @@ public override string ToString() return $"{GetName()}-{(int)_level}"; } } -} \ No newline at end of file +} diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/Avro.Codec.Zstandard.Test.csproj b/lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/Avro.Codec.Zstandard.Test.csproj new file mode 100644 index 00000000000..0df9b50c342 --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/Avro.Codec.Zstandard.Test.csproj @@ -0,0 +1,42 @@ + + + + + + + $(DefaultUnitTestTargetFrameworks) + false + + + + true + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/ZstandardTests.cs b/lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/ZstandardTests.cs new file mode 100644 index 00000000000..3189c830b42 --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/ZstandardTests.cs @@ -0,0 +1,37 @@ +using System.Linq; +using NUnit.Framework; + +namespace Avro.Codec.Zstandard.Test +{ + public class Tests + { + [TestCase(0)] + [TestCase(1000)] + [TestCase(64 * 1024)] + [TestCase(1 * 1024 * 1024)] + public void CompressDecompress(int length) + { + byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); + + ZstandardCodec codec = new ZstandardCodec(); + + byte[] compressed = codec.Compress(data); + byte[] uncompressed = codec.Decompress(compressed, compressed.Length); + + CollectionAssert.AreEqual(data, uncompressed); + } + + [Test] + [TestCase(ZstandardLevel.Level1, ExpectedResult = "zstandard[1]")] + [TestCase(ZstandardLevel.Level2, ExpectedResult = "zstandard[2]")] + [TestCase(ZstandardLevel.Level3, ExpectedResult = "zstandard[3]")] + public string ToStringAndName(ZstandardLevel level) + { + ZstandardCodec codec = new ZstandardCodec(level); + + Assert.AreEqual("zstandard", codec.GetName()); + + return codec.ToString(); + } + } +} diff --git a/lang/csharp/src/apache/codec/Zstandard/Avro.Codec.Zstandard.csproj b/lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Avro.Codec.Zstandard.csproj similarity index 100% rename from lang/csharp/src/apache/codec/Zstandard/Avro.Codec.Zstandard.csproj rename to lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Avro.Codec.Zstandard.csproj diff --git a/lang/csharp/src/apache/codec/Zstandard/Zstandard.cs b/lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Zstandard.cs similarity index 100% rename from lang/csharp/src/apache/codec/Zstandard/Zstandard.cs rename to lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Zstandard.cs diff --git a/lang/csharp/src/apache/codec/Avro.Codec.sln b/lang/csharp/src/apache/codec/Avro.Codec.sln index f06c28a1316..adcdc05f71c 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.sln +++ b/lang/csharp/src/apache/codec/Avro.Codec.sln @@ -1,26 +1,31 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.30114.105 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.32126.317 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Avro.Codec.BZip2", "BZip2\Avro.Codec.BZip2.csproj", "{40201607-AA97-44FB-8353-6CDFA803FBBB}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.Codec.BZip2", "Avro.Codec.BZip2\Avro.Codec.BZip2.csproj", "{40201607-AA97-44FB-8353-6CDFA803FBBB}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Avro.Codec.XZ", "XZ\Avro.Codec.XZ.csproj", "{06396DF0-D656-4133-8BDC-EBF194D13AE4}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.Codec.BZip2.Test", "Avro.Codec.BZip2.Test\Avro.Codec.BZip2.Test.csproj", "{ABEDA4C6-8024-48C1-AFC0-37190ABC659A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Avro.Codec.Snappy", "Snappy\Avro.Codec.Snappy.csproj", "{C3F05185-2D65-4A63-B16D-6BE20A254C1F}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.Codec.Snappy", "Avro.Codec.Snappy\Avro.Codec.Snappy.csproj", "{C3F05185-2D65-4A63-B16D-6BE20A254C1F}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Avro.main", "..\main\Avro.main.csproj", "{E5B71984-FAEC-43E2-B2AC-4413D737C5CD}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.Codec.Snappy.Test", "Avro.Codec.Snappy.Test\Avro.Codec.Snappy.Test.csproj", "{564A59B5-008D-4654-B63F-8EFAC8C45D10}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Avro.Codec.Zstandard", "Zstandard\Avro.Codec.Zstandard.csproj", "{21E6C150-200F-484C-949F-C818BBE4F276}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.Codec.XZ", "Avro.Codec.XZ\Avro.Codec.XZ.csproj", "{06396DF0-D656-4133-8BDC-EBF194D13AE4}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.Codec.XZ.Test", "Avro.Codec.XZ.Test\Avro.Codec.XZ.Test.csproj", "{4E8A6CCA-90DF-4350-B928-E6ADA5686883}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.Codec.Zstandard", "Avro.Codec.Zstandard\Avro.Codec.Zstandard.csproj", "{21E6C150-200F-484C-949F-C818BBE4F276}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.Codec.Zstandard.Test", "Avro.Codec.Zstandard.Test\Avro.Codec.Zstandard.Test.csproj", "{B93B1C26-C254-4C04-BB6B-E4FC8CCA9122}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.main", "..\main\Avro.main.csproj", "{E5B71984-FAEC-43E2-B2AC-4413D737C5CD}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU Release|Any CPU = Release|Any CPU EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {40201607-AA97-44FB-8353-6CDFA803FBBB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {40201607-AA97-44FB-8353-6CDFA803FBBB}.Debug|Any CPU.Build.0 = Debug|Any CPU @@ -42,5 +47,27 @@ Global {21E6C150-200F-484C-949F-C818BBE4F276}.Debug|Any CPU.Build.0 = Debug|Any CPU {21E6C150-200F-484C-949F-C818BBE4F276}.Release|Any CPU.ActiveCfg = Release|Any CPU {21E6C150-200F-484C-949F-C818BBE4F276}.Release|Any CPU.Build.0 = Release|Any CPU + {564A59B5-008D-4654-B63F-8EFAC8C45D10}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {564A59B5-008D-4654-B63F-8EFAC8C45D10}.Debug|Any CPU.Build.0 = Debug|Any CPU + {564A59B5-008D-4654-B63F-8EFAC8C45D10}.Release|Any CPU.ActiveCfg = Release|Any CPU + {564A59B5-008D-4654-B63F-8EFAC8C45D10}.Release|Any CPU.Build.0 = Release|Any CPU + {ABEDA4C6-8024-48C1-AFC0-37190ABC659A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {ABEDA4C6-8024-48C1-AFC0-37190ABC659A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {ABEDA4C6-8024-48C1-AFC0-37190ABC659A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {ABEDA4C6-8024-48C1-AFC0-37190ABC659A}.Release|Any CPU.Build.0 = Release|Any CPU + {4E8A6CCA-90DF-4350-B928-E6ADA5686883}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4E8A6CCA-90DF-4350-B928-E6ADA5686883}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4E8A6CCA-90DF-4350-B928-E6ADA5686883}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4E8A6CCA-90DF-4350-B928-E6ADA5686883}.Release|Any CPU.Build.0 = Release|Any CPU + {B93B1C26-C254-4C04-BB6B-E4FC8CCA9122}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B93B1C26-C254-4C04-BB6B-E4FC8CCA9122}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B93B1C26-C254-4C04-BB6B-E4FC8CCA9122}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B93B1C26-C254-4C04-BB6B-E4FC8CCA9122}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {32C95735-F692-433B-BCD2-A702CA701398} EndGlobalSection EndGlobal From a7ec7497a4b5d854596abd7ff0a0e05a78b766f3 Mon Sep 17 00:00:00 2001 From: Zoltan Csizmadia Date: Sat, 12 Feb 2022 09:39:46 -0600 Subject: [PATCH 03/28] Add license info --- .../codec/Avro.Codec.BZip2.Test/BZip2Tests.cs | 17 +++++++++++++++++ .../apache/codec/Avro.Codec.BZip2/BZip2.cs | 1 - .../Avro.Codec.Snappy.Test/SnappyTests.cs | 17 +++++++++++++++++ .../apache/codec/Avro.Codec.Snappy/Crc32.cs | 1 - .../apache/codec/Avro.Codec.Snappy/Snappy.cs | 1 - .../codec/Avro.Codec.XZ.Test/XZTests.cs | 17 +++++++++++++++++ .../src/apache/codec/Avro.Codec.XZ/XZ.cs | 1 - .../ZstandardTests.cs | 17 +++++++++++++++++ .../codec/Avro.Codec.Zstandard/Zstandard.cs | 1 - lang/csharp/src/apache/codec/Avro.Codec.sln | 19 +++++++++++++++++-- 10 files changed, 85 insertions(+), 7 deletions(-) diff --git a/lang/csharp/src/apache/codec/Avro.Codec.BZip2.Test/BZip2Tests.cs b/lang/csharp/src/apache/codec/Avro.Codec.BZip2.Test/BZip2Tests.cs index 0eb7b930295..cb2f6577519 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.BZip2.Test/BZip2Tests.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.BZip2.Test/BZip2Tests.cs @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ using System.Linq; using NUnit.Framework; diff --git a/lang/csharp/src/apache/codec/Avro.Codec.BZip2/BZip2.cs b/lang/csharp/src/apache/codec/Avro.Codec.BZip2/BZip2.cs index 3f2161e1fec..1f23413e4a5 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.BZip2/BZip2.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.BZip2/BZip2.cs @@ -15,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - using System; using System.IO; diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Snappy.Test/SnappyTests.cs b/lang/csharp/src/apache/codec/Avro.Codec.Snappy.Test/SnappyTests.cs index 7c35c27b43f..7f09c8f5bdf 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Snappy.Test/SnappyTests.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.Snappy.Test/SnappyTests.cs @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ using System.Linq; using NUnit.Framework; diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Crc32.cs b/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Crc32.cs index f450e8317c1..c9a9d3be552 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Crc32.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Crc32.cs @@ -15,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - using System; namespace Avro.Codec.Snappy diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Snappy.cs b/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Snappy.cs index 522440f7bd4..90093c9f39a 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Snappy.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Snappy.cs @@ -15,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - using System; using System.IO; using IronSnappy; diff --git a/lang/csharp/src/apache/codec/Avro.Codec.XZ.Test/XZTests.cs b/lang/csharp/src/apache/codec/Avro.Codec.XZ.Test/XZTests.cs index 7108c421b2b..9dbe9eb502a 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.XZ.Test/XZTests.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.XZ.Test/XZTests.cs @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ using System.Linq; using NUnit.Framework; diff --git a/lang/csharp/src/apache/codec/Avro.Codec.XZ/XZ.cs b/lang/csharp/src/apache/codec/Avro.Codec.XZ/XZ.cs index f3f4d4dbfcc..41ba29ff5e5 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.XZ/XZ.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.XZ/XZ.cs @@ -15,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - using System; using System.Collections.Generic; using System.IO; diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/ZstandardTests.cs b/lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/ZstandardTests.cs index 3189c830b42..87df1a46fbb 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/ZstandardTests.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/ZstandardTests.cs @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ using System.Linq; using NUnit.Framework; diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Zstandard.cs b/lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Zstandard.cs index 6978987ce73..bede583f106 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Zstandard.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Zstandard.cs @@ -15,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - using System; using System.IO; using ZstdNet; diff --git a/lang/csharp/src/apache/codec/Avro.Codec.sln b/lang/csharp/src/apache/codec/Avro.Codec.sln index adcdc05f71c..f5f24337d95 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.sln +++ b/lang/csharp/src/apache/codec/Avro.Codec.sln @@ -1,5 +1,20 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 +Microsoft Visual Studio Solution File, Format Version 12.00 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Visual Studio Version 17 VisualStudioVersion = 17.0.32126.317 MinimumVisualStudioVersion = 10.0.40219.1 From c28d9a4a08ba0262b9418777bb9fd86b7f285d72 Mon Sep 17 00:00:00 2001 From: Zoltan Csizmadia Date: Sat, 12 Feb 2022 10:05:48 -0600 Subject: [PATCH 04/28] Add codec build/test step --- lang/csharp/build.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lang/csharp/build.sh b/lang/csharp/build.sh index 6fe5cdf87a9..f72baa54f9f 100755 --- a/lang/csharp/build.sh +++ b/lang/csharp/build.sh @@ -38,6 +38,10 @@ do # AVRO-2442: Explicitly set LANG to work around ICU bug in `dotnet test` LANG=en_US.UTF-8 dotnet test --configuration Release --no-build \ --filter "TestCategory!=Interop" Avro.sln + + dotnet build --configuration Release src/apache/codec/Avro.Codec.sln + ANG=en_US.UTF-8 dotnet test --configuration Release --no-build \ + src/apache/codec/Avro.Codec.sln ;; perf) From d57fa6dde564f85ebcb4aa97245efb8a40d6c165 Mon Sep 17 00:00:00 2001 From: Zoltan Csizmadia Date: Sat, 12 Feb 2022 12:38:01 -0600 Subject: [PATCH 05/28] Install libzstd1 package --- .github/workflows/test-lang-csharp.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test-lang-csharp.yml b/.github/workflows/test-lang-csharp.yml index 81fafce1940..ebe1baea38e 100644 --- a/.github/workflows/test-lang-csharp.yml +++ b/.github/workflows/test-lang-csharp.yml @@ -34,6 +34,10 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Add libzstd + shell: bash + run: sudo apt-get install -y libzstd1 + - name: Install .NET SDKs uses: actions/setup-dotnet@v1 with: From 40e1a29074812e48796d2986b92f1ffa0603a0e2 Mon Sep 17 00:00:00 2001 From: Zoltan Csizmadia Date: Sat, 12 Feb 2022 12:40:08 -0600 Subject: [PATCH 06/28] Add libzstd-dev package --- .github/workflows/test-lang-csharp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-lang-csharp.yml b/.github/workflows/test-lang-csharp.yml index ebe1baea38e..39c5d6b50e2 100644 --- a/.github/workflows/test-lang-csharp.yml +++ b/.github/workflows/test-lang-csharp.yml @@ -36,7 +36,7 @@ jobs: - name: Add libzstd shell: bash - run: sudo apt-get install -y libzstd1 + run: sudo apt-get install -y libzstd-dev - name: Install .NET SDKs uses: actions/setup-dotnet@v1 From b9d4b1e0dac6fb91f3f56e58e8329baed939e60e Mon Sep 17 00:00:00 2001 From: Zoltan Csizmadia Date: Sun, 13 Feb 2022 10:59:14 -0600 Subject: [PATCH 07/28] Add stream compress unit tests --- .../codec/Avro.Codec.BZip2.Test/BZip2Tests.cs | 39 +++++++++++----- .../Avro.Codec.Snappy.Test/SnappyTests.cs | 27 +++++++++-- .../apache/codec/Avro.Codec.Snappy/Snappy.cs | 2 +- .../codec/Avro.Codec.XZ.Test/XZTests.cs | 45 +++++++++++-------- .../ZstandardTests.cs | 39 +++++++++++----- .../codec/Avro.Codec.Zstandard/Zstandard.cs | 20 +++++---- 6 files changed, 117 insertions(+), 55 deletions(-) diff --git a/lang/csharp/src/apache/codec/Avro.Codec.BZip2.Test/BZip2Tests.cs b/lang/csharp/src/apache/codec/Avro.Codec.BZip2.Test/BZip2Tests.cs index cb2f6577519..62447a36704 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.BZip2.Test/BZip2Tests.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.BZip2.Test/BZip2Tests.cs @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +using System.IO; using System.Linq; using NUnit.Framework; @@ -22,15 +23,14 @@ namespace Avro.Codec.BZip2.Test { public class Tests { - [TestCase(0)] - [TestCase(1000)] - [TestCase(64 * 1024)] - [TestCase(1 * 1024 * 1024)] - public void CompressDecompress(int length) + private static int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 1 * 1024 * 1024 }; + + [Test, Combinatorial] + public void CompressDecompress([ValueSource(nameof(_testLengths))] int length, [Values] BZip2Level level) { byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); - BZip2Codec codec = new BZip2Codec(); + BZip2Codec codec = new BZip2Codec(level); byte[] compressed = codec.Compress(data); byte[] uncompressed = codec.Decompress(compressed, compressed.Length); @@ -38,17 +38,32 @@ public void CompressDecompress(int length) CollectionAssert.AreEqual(data, uncompressed); } + [Test, Combinatorial] + public void CompressDecompressStream([ValueSource(nameof(_testLengths))] int length, [Values] BZip2Level level) + { + byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); + + BZip2Codec codec = new BZip2Codec(level); + + using (MemoryStream inputStream = new MemoryStream(data)) + using (MemoryStream outputStream = new MemoryStream()) + { + codec.Compress(inputStream, outputStream); + + byte[] compressed = outputStream.ToArray(); + byte[] uncompressed = codec.Decompress(compressed, compressed.Length); + + CollectionAssert.AreEqual(data, uncompressed); + } + } + [Test] - [TestCase(BZip2Level.Level1, ExpectedResult = "bzip2-1")] - [TestCase(BZip2Level.Level2, ExpectedResult = "bzip2-2")] - [TestCase(BZip2Level.Level3, ExpectedResult = "bzip2-3")] - public string ToStringAndName(BZip2Level level) + public void ToStringAndName([Values] BZip2Level level) { BZip2Codec codec = new BZip2Codec(level); Assert.AreEqual("bzip2", codec.GetName()); - - return codec.ToString(); + Assert.AreEqual($"bzip2-{(int)level}", codec.ToString()); } } } diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Snappy.Test/SnappyTests.cs b/lang/csharp/src/apache/codec/Avro.Codec.Snappy.Test/SnappyTests.cs index 7f09c8f5bdf..7bf31ab258b 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Snappy.Test/SnappyTests.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.Snappy.Test/SnappyTests.cs @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +using System.IO; using System.Linq; using NUnit.Framework; @@ -22,10 +23,9 @@ namespace Avro.Codec.Snappy.Test { public class Tests { - [TestCase(0)] - [TestCase(1000)] - [TestCase(64*1024)] - [TestCase(1*1024*1024)] + private static int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 1 * 1024 * 1024 }; + + [TestCaseSource(nameof(_testLengths))] public void CompressDecompress(int length) { byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); @@ -38,6 +38,25 @@ public void CompressDecompress(int length) CollectionAssert.AreEqual(data, uncompressed); } + [TestCaseSource(nameof(_testLengths))] + public void CompressDecompressStream(int length) + { + byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); + + SnappyCodec codec = new SnappyCodec(); + + using (MemoryStream inputStream = new MemoryStream(data)) + using (MemoryStream outputStream = new MemoryStream()) + { + codec.Compress(inputStream, outputStream); + + byte[] compressed = outputStream.ToArray(); + byte[] uncompressed = codec.Decompress(compressed, compressed.Length); + + CollectionAssert.AreEqual(data, uncompressed); + } + } + [Test] public void ToStringAndName() { diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Snappy.cs b/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Snappy.cs index 90093c9f39a..5b96e41e996 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Snappy.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Snappy.cs @@ -47,7 +47,7 @@ public override byte[] Compress(byte[] uncompressedData) /// public override void Compress(MemoryStream inputStream, MemoryStream outputStream) { - byte[] uncompressedData = inputStream.GetBuffer(); + byte[] uncompressedData = inputStream.ToArray(); byte[] compressedData = IronSnappy.Snappy.Encode(uncompressedData); outputStream.Write(compressedData, 0, compressedData.Length); diff --git a/lang/csharp/src/apache/codec/Avro.Codec.XZ.Test/XZTests.cs b/lang/csharp/src/apache/codec/Avro.Codec.XZ.Test/XZTests.cs index 9dbe9eb502a..46daecc57d5 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.XZ.Test/XZTests.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.XZ.Test/XZTests.cs @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +using System.IO; using System.Linq; using NUnit.Framework; @@ -22,27 +23,20 @@ namespace Avro.Codec.XZ.Test { public class Tests { + private static int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 1 * 1024 * 1024 }; + [OneTimeSetUp] public void OneTimeSetup() { XZCodec.Initialize(); } - [OneTimeTearDown] - public void OneTimeTearDOwn() - { - XZCodec.Uninitialize(); - } - - [TestCase(0)] - [TestCase(1000)] - [TestCase(64 * 1024)] - [TestCase(1 * 1024 * 1024)] - public void CompressDecompress(int length) + [Test, Combinatorial] + public void CompressDecompress([ValueSource(nameof(_testLengths))] int length, [Values] XZLevel level) { byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); - XZCodec codec = new XZCodec(); + XZCodec codec = new XZCodec(level); byte[] compressed = codec.Compress(data); byte[] uncompressed = codec.Decompress(compressed, compressed.Length); @@ -50,17 +44,32 @@ public void CompressDecompress(int length) CollectionAssert.AreEqual(data, uncompressed); } + [Test, Combinatorial] + public void CompressDecompressStream([ValueSource(nameof(_testLengths))] int length, [Values] XZLevel level) + { + byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); + + XZCodec codec = new XZCodec(level); + + using (MemoryStream inputStream = new MemoryStream(data)) + using (MemoryStream outputStream = new MemoryStream()) + { + codec.Compress(inputStream, outputStream); + + byte[] compressed = outputStream.ToArray(); + byte[] uncompressed = codec.Decompress(compressed, compressed.Length); + + CollectionAssert.AreEqual(data, uncompressed); + } + } + [Test] - [TestCase(XZLevel.Level1, ExpectedResult = "xz-1")] - [TestCase(XZLevel.Level2, ExpectedResult = "xz-2")] - [TestCase(XZLevel.Level3, ExpectedResult = "xz-3")] - public string ToStringAndName(XZLevel level) + public void ToStringAndName([Values] XZLevel level) { XZCodec codec = new XZCodec(level); Assert.AreEqual("xz", codec.GetName()); - - return codec.ToString(); + Assert.AreEqual($"xz-{(int)level}", codec.ToString()); } } } diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/ZstandardTests.cs b/lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/ZstandardTests.cs index 87df1a46fbb..60d59033407 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/ZstandardTests.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/ZstandardTests.cs @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +using System.IO; using System.Linq; using NUnit.Framework; @@ -22,15 +23,14 @@ namespace Avro.Codec.Zstandard.Test { public class Tests { - [TestCase(0)] - [TestCase(1000)] - [TestCase(64 * 1024)] - [TestCase(1 * 1024 * 1024)] - public void CompressDecompress(int length) + private static int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 1 * 1024 * 1024 }; + + [Test, Combinatorial] + public void CompressDecompress([ValueSource(nameof(_testLengths))] int length, [Values] ZstandardLevel level) { byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); - ZstandardCodec codec = new ZstandardCodec(); + ZstandardCodec codec = new ZstandardCodec(level); byte[] compressed = codec.Compress(data); byte[] uncompressed = codec.Decompress(compressed, compressed.Length); @@ -38,17 +38,32 @@ public void CompressDecompress(int length) CollectionAssert.AreEqual(data, uncompressed); } + [Test, Combinatorial] + public void CompressDecompressStream([ValueSource(nameof(_testLengths))] int length, [Values] ZstandardLevel level) + { + byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); + + ZstandardCodec codec = new ZstandardCodec(level); + + using (MemoryStream inputStream = new MemoryStream(data)) + using (MemoryStream outputStream = new MemoryStream()) + { + codec.Compress(inputStream, outputStream); + + byte[] compressed = outputStream.ToArray(); + byte[] uncompressed = codec.Decompress(compressed, compressed.Length); + + CollectionAssert.AreEqual(data, uncompressed); + } + } + [Test] - [TestCase(ZstandardLevel.Level1, ExpectedResult = "zstandard[1]")] - [TestCase(ZstandardLevel.Level2, ExpectedResult = "zstandard[2]")] - [TestCase(ZstandardLevel.Level3, ExpectedResult = "zstandard[3]")] - public string ToStringAndName(ZstandardLevel level) + public void ToStringAndName([Values] ZstandardLevel level) { ZstandardCodec codec = new ZstandardCodec(level); Assert.AreEqual("zstandard", codec.GetName()); - - return codec.ToString(); + Assert.AreEqual($"zstandard[{(int)level}]", codec.ToString()); } } } diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Zstandard.cs b/lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Zstandard.cs index bede583f106..fcf14a0f702 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Zstandard.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Zstandard.cs @@ -18,6 +18,7 @@ using System; using System.IO; using ZstdNet; +using System.IO.Compression; namespace Avro.Codec.Zstandard { @@ -72,8 +73,8 @@ public ZstandardCodec(ZstandardLevel level) /// public override byte[] Compress(byte[] uncompressedData) { - using(CompressionOptions options = new CompressionOptions((int)_level)) - using(Compressor compressor = new Compressor(options)) + using (CompressionOptions options = new CompressionOptions((int)_level)) + using (Compressor compressor = new Compressor(options)) { return compressor.Wrap(uncompressedData); } @@ -82,11 +83,14 @@ public override byte[] Compress(byte[] uncompressedData) /// public override void Compress(MemoryStream inputStream, MemoryStream outputStream) { - using(CompressionOptions options = new CompressionOptions((int)_level)) - using(CompressionStream compressionStream = new CompressionStream(outputStream, options)) - { - inputStream.CopyTo(compressionStream); - } + byte[] compressedData = Compress(inputStream.ToArray()); + outputStream.Write(compressedData, 0, compressedData.Length); + + //using (CompressionOptions options = new CompressionOptions((int)_level)) + //using (CompressionStream compressionStream = new CompressionStream(outputStream, options)) + //{ + // inputStream.CopyTo(compressionStream); + //} } /// @@ -122,4 +126,4 @@ public override string ToString() return $"{GetName()}[{(int)_level}]"; } } -} \ No newline at end of file +} From 5073061db450efc27645f222ddf0b99b25fff759 Mon Sep 17 00:00:00 2001 From: Zoltan Csizmadia Date: Sun, 13 Feb 2022 10:59:28 -0600 Subject: [PATCH 08/28] Add brotli --- .../Avro.Codec.Brotli.Test.csproj | 42 +++++++ .../Avro.Codec.Brotli.Test/BrotliTests.cs | 69 +++++++++++ .../Avro.Codec.Brotli.csproj | 47 ++++++++ .../apache/codec/Avro.Codec.Brotli/Brotli.cs | 110 ++++++++++++++++++ lang/csharp/src/apache/codec/Avro.Codec.sln | 47 +++++--- 5 files changed, 298 insertions(+), 17 deletions(-) create mode 100644 lang/csharp/src/apache/codec/Avro.Codec.Brotli.Test/Avro.Codec.Brotli.Test.csproj create mode 100644 lang/csharp/src/apache/codec/Avro.Codec.Brotli.Test/BrotliTests.cs create mode 100644 lang/csharp/src/apache/codec/Avro.Codec.Brotli/Avro.Codec.Brotli.csproj create mode 100644 lang/csharp/src/apache/codec/Avro.Codec.Brotli/Brotli.cs diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Brotli.Test/Avro.Codec.Brotli.Test.csproj b/lang/csharp/src/apache/codec/Avro.Codec.Brotli.Test/Avro.Codec.Brotli.Test.csproj new file mode 100644 index 00000000000..68e8e296c9e --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.Codec.Brotli.Test/Avro.Codec.Brotli.Test.csproj @@ -0,0 +1,42 @@ + + + + + + + $(DefaultUnitTestTargetFrameworks) + false + + + + true + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Brotli.Test/BrotliTests.cs b/lang/csharp/src/apache/codec/Avro.Codec.Brotli.Test/BrotliTests.cs new file mode 100644 index 00000000000..e30653b8460 --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.Codec.Brotli.Test/BrotliTests.cs @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using System.IO; +using System.Linq; +using NUnit.Framework; + +namespace Avro.Codec.Brotli.Test +{ + public class Tests + { + private static int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 1 * 1024 * 1024 }; + + [Test, Combinatorial] + public void CompressDecompress([ValueSource(nameof(_testLengths))] int length, [Values] BrotliLevel level) + { + byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); + + BrotliCodec codec = new BrotliCodec(level); + + byte[] compressed = codec.Compress(data); + byte[] uncompressed = codec.Decompress(compressed, compressed.Length); + + CollectionAssert.AreEqual(data, uncompressed); + } + + [Test, Combinatorial] + public void CompressDecompressStream([ValueSource(nameof(_testLengths))] int length, [Values] BrotliLevel level) + { + byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); + + BrotliCodec codec = new BrotliCodec(level); + + using (MemoryStream inputStream = new MemoryStream(data)) + using (MemoryStream outputStream = new MemoryStream()) + { + codec.Compress(inputStream, outputStream); + + byte[] compressed = outputStream.ToArray(); + byte[] uncompressed = codec.Decompress(compressed, compressed.Length); + + CollectionAssert.AreEqual(data, uncompressed); + } + } + + [Test] + public void ToStringAndName([Values] BrotliLevel level) + { + BrotliCodec codec = new BrotliCodec(level); + + Assert.AreEqual("brotli", codec.GetName()); + Assert.AreEqual($"brotli-{(int)level}", codec.ToString()); + } + } +} diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Brotli/Avro.Codec.Brotli.csproj b/lang/csharp/src/apache/codec/Avro.Codec.Brotli/Avro.Codec.Brotli.csproj new file mode 100644 index 00000000000..05726d7d359 --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.Codec.Brotli/Avro.Codec.Brotli.csproj @@ -0,0 +1,47 @@ + + + + + + + $(DefaultLibraryTargetFrameworks) + Avro.Codec.Brotli + true + ../../../../Avro.snk + + + + + Apache.Avro.Codec.Brotli + + + + + true + + + + + + + + + + + + diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Brotli/Brotli.cs b/lang/csharp/src/apache/codec/Avro.Codec.Brotli/Brotli.cs new file mode 100644 index 00000000000..235f7d9f5e4 --- /dev/null +++ b/lang/csharp/src/apache/codec/Avro.Codec.Brotli/Brotli.cs @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +using System; +using System.IO; +using System.IO.Compression; +using BrotliSharpLib; + +namespace Avro.Codec.Brotli +{ + /// + /// Brotli Compression level + /// + public enum BrotliLevel + { + Default = 1, + Level0 = 0, + Level1 = 1, + Level2 = 2, + Level3 = 3, + Level4 = 4, + Level5 = 5, + Level6 = 6, + Level7 = 7, + Level8 = 8, + Level9 = 9, + Level10 = 10, + Level11 = 11 + } + + /// + /// Implements Brotli compression and decompression. + /// + public class BrotliCodec : Avro.File.Codec + { + public const string DataFileConstant = "brotli"; + + private BrotliLevel _level; + + public BrotliCodec() + : this(BrotliLevel.Default) + { + } + + public BrotliCodec(BrotliLevel level) + { + _level = level; + } + + /// + public override byte[] Compress(byte[] uncompressedData) + { + return BrotliSharpLib.Brotli.CompressBuffer(uncompressedData, 0, uncompressedData.Length, (int)_level); + } + + /// + public override void Compress(MemoryStream inputStream, MemoryStream outputStream) + { + using (BrotliSharpLib.BrotliStream brotliStreams = new BrotliSharpLib.BrotliStream(outputStream, CompressionMode.Compress, false)) + { + brotliStreams.SetQuality((int)_level); + inputStream.CopyTo(brotliStreams); + } + } + + /// + public override byte[] Decompress(byte[] compressedData, int blockLength) + { + return BrotliSharpLib.Brotli.DecompressBuffer(compressedData, 0, blockLength); + } + + /// + public override string GetName() + { + return DataFileConstant; + } + + /// + public override bool Equals(object other) + { + return this == other || GetType().Name == other.GetType().Name; + } + + /// + public override int GetHashCode() + { + return 0; + } + + /// + public override string ToString() + { + return $"{GetName()}-{(int)_level}"; + } + } +} diff --git a/lang/csharp/src/apache/codec/Avro.Codec.sln b/lang/csharp/src/apache/codec/Avro.Codec.sln index f5f24337d95..20e5afc268a 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.sln +++ b/lang/csharp/src/apache/codec/Avro.Codec.sln @@ -1,4 +1,5 @@ -Microsoft Visual Studio Solution File, Format Version 12.00 + +Microsoft Visual Studio Solution File, Format Version 12.00 # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -36,6 +37,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.Codec.Zstandard.Test", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.main", "..\main\Avro.main.csproj", "{E5B71984-FAEC-43E2-B2AC-4413D737C5CD}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Avro.Codec.Brotli", "Avro.Codec.Brotli\Avro.Codec.Brotli.csproj", "{9174B8A2-FBFF-4992-BF15-CE5729F695A7}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.Codec.Brotli.Test", "Avro.Codec.Brotli.Test\Avro.Codec.Brotli.Test.csproj", "{9CBB2B45-DEE4-432D-8841-F8524F1F4F9A}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -46,38 +51,46 @@ Global {40201607-AA97-44FB-8353-6CDFA803FBBB}.Debug|Any CPU.Build.0 = Debug|Any CPU {40201607-AA97-44FB-8353-6CDFA803FBBB}.Release|Any CPU.ActiveCfg = Release|Any CPU {40201607-AA97-44FB-8353-6CDFA803FBBB}.Release|Any CPU.Build.0 = Release|Any CPU - {06396DF0-D656-4133-8BDC-EBF194D13AE4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {06396DF0-D656-4133-8BDC-EBF194D13AE4}.Debug|Any CPU.Build.0 = Debug|Any CPU - {06396DF0-D656-4133-8BDC-EBF194D13AE4}.Release|Any CPU.ActiveCfg = Release|Any CPU - {06396DF0-D656-4133-8BDC-EBF194D13AE4}.Release|Any CPU.Build.0 = Release|Any CPU + {ABEDA4C6-8024-48C1-AFC0-37190ABC659A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {ABEDA4C6-8024-48C1-AFC0-37190ABC659A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {ABEDA4C6-8024-48C1-AFC0-37190ABC659A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {ABEDA4C6-8024-48C1-AFC0-37190ABC659A}.Release|Any CPU.Build.0 = Release|Any CPU {C3F05185-2D65-4A63-B16D-6BE20A254C1F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {C3F05185-2D65-4A63-B16D-6BE20A254C1F}.Debug|Any CPU.Build.0 = Debug|Any CPU {C3F05185-2D65-4A63-B16D-6BE20A254C1F}.Release|Any CPU.ActiveCfg = Release|Any CPU {C3F05185-2D65-4A63-B16D-6BE20A254C1F}.Release|Any CPU.Build.0 = Release|Any CPU - {E5B71984-FAEC-43E2-B2AC-4413D737C5CD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {E5B71984-FAEC-43E2-B2AC-4413D737C5CD}.Debug|Any CPU.Build.0 = Debug|Any CPU - {E5B71984-FAEC-43E2-B2AC-4413D737C5CD}.Release|Any CPU.ActiveCfg = Release|Any CPU - {E5B71984-FAEC-43E2-B2AC-4413D737C5CD}.Release|Any CPU.Build.0 = Release|Any CPU - {21E6C150-200F-484C-949F-C818BBE4F276}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {21E6C150-200F-484C-949F-C818BBE4F276}.Debug|Any CPU.Build.0 = Debug|Any CPU - {21E6C150-200F-484C-949F-C818BBE4F276}.Release|Any CPU.ActiveCfg = Release|Any CPU - {21E6C150-200F-484C-949F-C818BBE4F276}.Release|Any CPU.Build.0 = Release|Any CPU {564A59B5-008D-4654-B63F-8EFAC8C45D10}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {564A59B5-008D-4654-B63F-8EFAC8C45D10}.Debug|Any CPU.Build.0 = Debug|Any CPU {564A59B5-008D-4654-B63F-8EFAC8C45D10}.Release|Any CPU.ActiveCfg = Release|Any CPU {564A59B5-008D-4654-B63F-8EFAC8C45D10}.Release|Any CPU.Build.0 = Release|Any CPU - {ABEDA4C6-8024-48C1-AFC0-37190ABC659A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {ABEDA4C6-8024-48C1-AFC0-37190ABC659A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {ABEDA4C6-8024-48C1-AFC0-37190ABC659A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {ABEDA4C6-8024-48C1-AFC0-37190ABC659A}.Release|Any CPU.Build.0 = Release|Any CPU + {06396DF0-D656-4133-8BDC-EBF194D13AE4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {06396DF0-D656-4133-8BDC-EBF194D13AE4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {06396DF0-D656-4133-8BDC-EBF194D13AE4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {06396DF0-D656-4133-8BDC-EBF194D13AE4}.Release|Any CPU.Build.0 = Release|Any CPU {4E8A6CCA-90DF-4350-B928-E6ADA5686883}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {4E8A6CCA-90DF-4350-B928-E6ADA5686883}.Debug|Any CPU.Build.0 = Debug|Any CPU {4E8A6CCA-90DF-4350-B928-E6ADA5686883}.Release|Any CPU.ActiveCfg = Release|Any CPU {4E8A6CCA-90DF-4350-B928-E6ADA5686883}.Release|Any CPU.Build.0 = Release|Any CPU + {21E6C150-200F-484C-949F-C818BBE4F276}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {21E6C150-200F-484C-949F-C818BBE4F276}.Debug|Any CPU.Build.0 = Debug|Any CPU + {21E6C150-200F-484C-949F-C818BBE4F276}.Release|Any CPU.ActiveCfg = Release|Any CPU + {21E6C150-200F-484C-949F-C818BBE4F276}.Release|Any CPU.Build.0 = Release|Any CPU {B93B1C26-C254-4C04-BB6B-E4FC8CCA9122}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B93B1C26-C254-4C04-BB6B-E4FC8CCA9122}.Debug|Any CPU.Build.0 = Debug|Any CPU {B93B1C26-C254-4C04-BB6B-E4FC8CCA9122}.Release|Any CPU.ActiveCfg = Release|Any CPU {B93B1C26-C254-4C04-BB6B-E4FC8CCA9122}.Release|Any CPU.Build.0 = Release|Any CPU + {E5B71984-FAEC-43E2-B2AC-4413D737C5CD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E5B71984-FAEC-43E2-B2AC-4413D737C5CD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E5B71984-FAEC-43E2-B2AC-4413D737C5CD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E5B71984-FAEC-43E2-B2AC-4413D737C5CD}.Release|Any CPU.Build.0 = Release|Any CPU + {9174B8A2-FBFF-4992-BF15-CE5729F695A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9174B8A2-FBFF-4992-BF15-CE5729F695A7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9174B8A2-FBFF-4992-BF15-CE5729F695A7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9174B8A2-FBFF-4992-BF15-CE5729F695A7}.Release|Any CPU.Build.0 = Release|Any CPU + {9CBB2B45-DEE4-432D-8841-F8524F1F4F9A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9CBB2B45-DEE4-432D-8841-F8524F1F4F9A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9CBB2B45-DEE4-432D-8841-F8524F1F4F9A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9CBB2B45-DEE4-432D-8841-F8524F1F4F9A}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 242ebf4e534fcf15395318f24c5ff12b55a9fe36 Mon Sep 17 00:00:00 2001 From: Zoltan Csizmadia Date: Sun, 13 Feb 2022 13:42:52 -0600 Subject: [PATCH 09/28] Change comment --- .../src/apache/codec/Avro.Codec.Zstandard/Zstandard.cs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Zstandard.cs b/lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Zstandard.cs index fcf14a0f702..9862028bd40 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Zstandard.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Zstandard.cs @@ -52,7 +52,7 @@ public enum ZstandardLevel } /// - /// Implements BZip2 compression and decompression. + /// Implements Zstandard compression and decompression. /// public class ZstandardCodec : Avro.File.Codec { @@ -85,12 +85,6 @@ public override void Compress(MemoryStream inputStream, MemoryStream outputStrea { byte[] compressedData = Compress(inputStream.ToArray()); outputStream.Write(compressedData, 0, compressedData.Length); - - //using (CompressionOptions options = new CompressionOptions((int)_level)) - //using (CompressionStream compressionStream = new CompressionStream(outputStream, options)) - //{ - // inputStream.CopyTo(compressionStream); - //} } /// From a2505830fd69d2ed7720be72e4c021c83fa469c6 Mon Sep 17 00:00:00 2001 From: Zoltan Csizmadia Date: Mon, 14 Feb 2022 09:55:40 -0600 Subject: [PATCH 10/28] Minor cleanups --- .../apache/codec/Avro.Codec.BZip2/BZip2.cs | 6 +- .../apache/codec/Avro.Codec.Brotli/Brotli.cs | 11 ++- .../apache/codec/Avro.Codec.Snappy/Crc32.cs | 10 +-- .../apache/codec/Avro.Codec.Snappy/Snappy.cs | 14 ++-- .../src/apache/codec/Avro.Codec.XZ/XZ.cs | 80 +++++++++---------- .../codec/Avro.Codec.Zstandard/Zstandard.cs | 9 ++- 6 files changed, 61 insertions(+), 69 deletions(-) diff --git a/lang/csharp/src/apache/codec/Avro.Codec.BZip2/BZip2.cs b/lang/csharp/src/apache/codec/Avro.Codec.BZip2/BZip2.cs index 1f23413e4a5..eabc01ee8fd 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.BZip2/BZip2.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.BZip2/BZip2.cs @@ -15,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -using System; using System.IO; namespace Avro.Codec.BZip2 @@ -40,11 +39,11 @@ public enum BZip2Level /// /// Implements BZip2 compression and decompression. /// - public class BZip2Codec : Avro.File.Codec + public class BZip2Codec : File.Codec { public const string DataFileConstant = "bzip2"; - private BZip2Level _level; + private readonly BZip2Level _level; public BZip2Codec() : this(BZip2Level.Default) @@ -70,6 +69,7 @@ public override byte[] Compress(byte[] uncompressedData) /// public override void Compress(MemoryStream inputStream, MemoryStream outputStream) { + outputStream.SetLength(0); ICSharpCode.SharpZipLib.BZip2.BZip2.Compress(inputStream, outputStream, false, (int)_level); } diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Brotli/Brotli.cs b/lang/csharp/src/apache/codec/Avro.Codec.Brotli/Brotli.cs index 235f7d9f5e4..76c8947c575 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Brotli/Brotli.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.Brotli/Brotli.cs @@ -1,4 +1,4 @@ -/* +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,9 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -using System; using System.IO; -using System.IO.Compression; using BrotliSharpLib; namespace Avro.Codec.Brotli @@ -45,11 +43,11 @@ public enum BrotliLevel /// /// Implements Brotli compression and decompression. /// - public class BrotliCodec : Avro.File.Codec + public class BrotliCodec : File.Codec { public const string DataFileConstant = "brotli"; - private BrotliLevel _level; + private readonly BrotliLevel _level; public BrotliCodec() : this(BrotliLevel.Default) @@ -70,7 +68,8 @@ public override byte[] Compress(byte[] uncompressedData) /// public override void Compress(MemoryStream inputStream, MemoryStream outputStream) { - using (BrotliSharpLib.BrotliStream brotliStreams = new BrotliSharpLib.BrotliStream(outputStream, CompressionMode.Compress, false)) + outputStream.SetLength(0); + using (BrotliStream brotliStreams = new BrotliStream(outputStream, System.IO.Compression.CompressionMode.Compress, false)) { brotliStreams.SetQuality((int)_level); inputStream.CopyTo(brotliStreams); diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Crc32.cs b/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Crc32.cs index c9a9d3be552..c5fad481efe 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Crc32.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Crc32.cs @@ -24,10 +24,10 @@ namespace Avro.Codec.Snappy /// internal static class Crc32 { - const uint DefaultPolynomial = 0xedb88320u; - const uint DefaultSeed = 0xffffffffu; + private const uint DefaultPolynomial = 0xedb88320u; + private const uint DefaultSeed = 0xffffffffu; - static uint[] defaultTable; + private static uint[] defaultTable; public static uint Compute(byte[] buffer) { @@ -39,7 +39,7 @@ public static uint Compute(uint polynomial, uint seed, ReadOnlySpan buffer return ~CalculateHash(InitializeTable(polynomial), seed, buffer); } - static uint[] InitializeTable(uint polynomial) + private static uint[] InitializeTable(uint polynomial) { if (polynomial == DefaultPolynomial && defaultTable != null) return defaultTable; @@ -62,7 +62,7 @@ static uint[] InitializeTable(uint polynomial) return createTable; } - static uint CalculateHash(uint[] table, uint seed, ReadOnlySpan buffer) + private static uint CalculateHash(uint[] table, uint seed, ReadOnlySpan buffer) { uint hash = seed; for (int i = 0; i < buffer.Length; i++) diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Snappy.cs b/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Snappy.cs index 5b96e41e996..5d815d9bf58 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Snappy.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Snappy.cs @@ -17,14 +17,13 @@ */ using System; using System.IO; -using IronSnappy; namespace Avro.Codec.Snappy { /// /// Implements Snappy compression and decompression. /// - public class SnappyCodec : Avro.File.Codec + public class SnappyCodec : File.Codec { public const string DataFileConstant = "snappy"; @@ -49,6 +48,9 @@ public override void Compress(MemoryStream inputStream, MemoryStream outputStrea { byte[] uncompressedData = inputStream.ToArray(); byte[] compressedData = IronSnappy.Snappy.Encode(uncompressedData); + + outputStream.SetLength(0); + outputStream.Write(compressedData, 0, compressedData.Length); var crc = ByteSwap(Crc32.Compute(uncompressedData)); @@ -60,13 +62,9 @@ public override byte[] Decompress(byte[] compressedData, int blockLength) { byte[] uncompressedData = IronSnappy.Snappy.Decode(compressedData.AsSpan(0, blockLength - 4)); - var crc = ByteSwap(Crc32.Compute(uncompressedData)); - if (crc != BitConverter.ToUInt32(compressedData, blockLength - 4)) - { + return ByteSwap(Crc32.Compute(uncompressedData)) == BitConverter.ToUInt32(compressedData, blockLength - 4) ? + uncompressedData : throw new IOException("Checksum failure"); - } - - return uncompressedData; } private static uint ByteSwap(uint word) diff --git a/lang/csharp/src/apache/codec/Avro.Codec.XZ/XZ.cs b/lang/csharp/src/apache/codec/Avro.Codec.XZ/XZ.cs index 41ba29ff5e5..ae22c45ad67 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.XZ/XZ.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.XZ/XZ.cs @@ -46,13 +46,13 @@ public enum XZLevel /// /// Implements XZ compression and decompression. /// - public class XZCodec : Avro.File.Codec + public class XZCodec : File.Codec { public const string DataFileConstant = "xz"; - private XZLevel _level; - private bool _extreme; - private int _threads; + private readonly XZLevel _level; + private readonly bool _extreme; + private readonly int _threads; public XZCodec() : this(XZLevel.Default) @@ -82,58 +82,35 @@ public XZCodec(XZLevel level, bool extreme, int numOfThreads) // !!! public static void Initialize() { - string libPath; + string arch = RuntimeInformation.OSArchitecture.ToString().ToLower(); string foundLibPath = string.Empty; - string libName = "liblzma"; + string libPath; string rid; - string arch; - - switch(RuntimeInformation.OSArchitecture) - { - case Architecture.X86: - arch = "x86"; - break; - case Architecture.X64: - arch = "x64"; - break; - case Architecture.Arm: - arch = "arm"; - break; - case Architecture.Arm64: - arch = "arm64"; - break; - default: - throw new Exception("Unknown runtime architecture!"); - }; - - // Unknown architecture - if (arch == null) - throw new Exception("Unknown runtime architecture!"); + string libName; // Determine Platform (needed for proper Runtime ID) if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { rid = $"win-{arch}"; + libName = "liblzma.dll"; + } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { rid = $"linux-{arch}"; + libName = "liblzma.so"; + } else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { rid = $"osx-{arch}"; + libName = "liblzma.dylib"; + } else + { // Unknown platform - throw new Exception("Unknown runtime platform!"); - - // Determine Platform (needed for proper Runtime ID) - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - libName += ".dll"; - else - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - libName += ".so"; - else - if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - libName += ".dylib"; - else - throw new PlatformNotSupportedException(); + throw new PlatformNotSupportedException("Unknown runtime platform!"); + } // Try to search for the lib in the working directory and the application binary directory foreach (var relPath in new List { ".", AppDomain.CurrentDomain.BaseDirectory }) @@ -156,8 +133,23 @@ public static void Initialize() } } + // Try the OS search path if nothing is found yet + if (string.IsNullOrEmpty(foundLibPath)) + { + var values = Environment.GetEnvironmentVariable("PATH"); + foreach (string path in values.Split(Path.PathSeparator)) + { + libPath = Path.Combine(path, libName); + if (System.IO.File.Exists(libPath)) + { + foundLibPath = libPath; + break; + } + } + } + if (string.IsNullOrEmpty(foundLibPath)) - throw new PlatformNotSupportedException(); + throw new PlatformNotSupportedException($"Unable to find {libName}"); // Initialize XZ library XZInit.GlobalInit(foundLibPath); @@ -194,6 +186,8 @@ public override void Compress(MemoryStream inputStream, MemoryStream outputStrea Threads = _threads, }; + outputStream.SetLength(0); + using (XZStream xzStream = new XZStream(outputStream, compOpts, threadOpts)) { inputStream.CopyTo(xzStream); diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Zstandard.cs b/lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Zstandard.cs index 9862028bd40..2d49f4a6586 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Zstandard.cs +++ b/lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Zstandard.cs @@ -18,7 +18,6 @@ using System; using System.IO; using ZstdNet; -using System.IO.Compression; namespace Avro.Codec.Zstandard { @@ -54,11 +53,11 @@ public enum ZstandardLevel /// /// Implements Zstandard compression and decompression. /// - public class ZstandardCodec : Avro.File.Codec + public class ZstandardCodec : File.Codec { public const string DataFileConstant = "zstandard"; - private ZstandardLevel _level; + private readonly ZstandardLevel _level; public ZstandardCodec() : this(ZstandardLevel.Default) @@ -84,13 +83,15 @@ public override byte[] Compress(byte[] uncompressedData) public override void Compress(MemoryStream inputStream, MemoryStream outputStream) { byte[] compressedData = Compress(inputStream.ToArray()); + + outputStream.SetLength(0); outputStream.Write(compressedData, 0, compressedData.Length); } /// public override byte[] Decompress(byte[] compressedData, int blockLength) { - using(Decompressor decompressor = new Decompressor()) + using (Decompressor decompressor = new Decompressor()) { return decompressor.Unwrap(compressedData.AsSpan(0, blockLength)); } From 85c5a8c6c57dc3dbbdb965fe058b8f49947f1a4b Mon Sep 17 00:00:00 2001 From: Zoltan Csizmadia Date: Mon, 14 Feb 2022 12:36:51 -0600 Subject: [PATCH 11/28] Add libzstd-dev --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index a9967017447..c51e3a04aca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -94,6 +94,7 @@ matrix: update: true packages: - wget + - libzstd-dev install: - wget https://dot.net/v1/dotnet-install.sh - bash ./dotnet-install.sh --version "3.1.402" --install-dir "$HOME/.dotnet" --architecture "arm64" --os "linux" # 3.1 From c2b6fccd1dbf1aa3a3f7f812aa5756b96bf73d3e Mon Sep 17 00:00:00 2001 From: Zoltan Csizmadia Date: Tue, 15 Feb 2022 16:10:21 -0600 Subject: [PATCH 12/28] Dynamically create extra codecs --- lang/csharp/src/apache/main/File/Codec.cs | 38 +++++++++++++++++++ .../src/apache/main/File/DataFileConstants.cs | 20 ++++++++++ 2 files changed, 58 insertions(+) diff --git a/lang/csharp/src/apache/main/File/Codec.cs b/lang/csharp/src/apache/main/File/Codec.cs index 3a5b27bea47..a36ac34a215 100644 --- a/lang/csharp/src/apache/main/File/Codec.cs +++ b/lang/csharp/src/apache/main/File/Codec.cs @@ -19,6 +19,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Reflection; namespace Avro.File { @@ -101,6 +102,26 @@ public enum Type /// Codec that does not perform any compression. /// Null, + + /// + /// Codec type that implements the "Snappy" compression algorithm. + /// + Snappy, + + /// + /// Codec type that implements the "BZip2" compression algorithm. + /// + BZip2, + + /// + /// Codec type that implements the "XZ" compression algorithm. + /// + XZ, + + /// + /// Codec type that implements the "Zstandard" compression algorithm. + /// + Zstandard } /// @@ -140,6 +161,15 @@ public static Codec CreateCodec(Type codecType) return new DeflateCodec(); case Type.Null: return new NullCodec(); + case Type.Snappy: + case Type.BZip2: + case Type.XZ: + case Type.Zstandard: + { + // Create codec dynamically from "Avro.Codec.CODECNAME" assembly + Assembly assembly = Assembly.Load($"Avro.Codec.{codecType}"); + return assembly.CreateInstance($"Avro.Codec.{codecType}.{codecType}Codec") as Codec; + } } throw new AvroRuntimeException($"Unrecognized codec: {codecType}"); @@ -174,6 +204,14 @@ public static Codec CreateCodecFromString(string codecType) return CreateCodec(Type.Deflate); case DataFileConstants.NullCodec: return CreateCodec(Type.Null); + case DataFileConstants.SnappyCodec: + return CreateCodec(Type.Snappy); + case DataFileConstants.BZip2Codec: + return CreateCodec(Type.BZip2); + case DataFileConstants.XZCodec: + return CreateCodec(Type.XZ); + case DataFileConstants.ZstandardCodec: + return CreateCodec(Type.Zstandard); } throw new AvroRuntimeException($"Unrecognized codec: {codecType}"); diff --git a/lang/csharp/src/apache/main/File/DataFileConstants.cs b/lang/csharp/src/apache/main/File/DataFileConstants.cs index b99e0a4447f..438caeca0e4 100644 --- a/lang/csharp/src/apache/main/File/DataFileConstants.cs +++ b/lang/csharp/src/apache/main/File/DataFileConstants.cs @@ -48,6 +48,26 @@ public class DataFileConstants /// public const string DeflateCodec = "deflate"; + /// + /// Identifier for the Snappy codec. + /// + public const string SnappyCodec = "snappy"; + + /// + /// Identifier for the BZip2 codec. + /// + public const string BZip2Codec = "bzip2"; + + /// + /// Identifier for the XZ codec. + /// + public const string XZCodec = "xz"; + + /// + /// Identifier for the Zstandard codec. + /// + public const string ZstandardCodec = "zstandard"; + /// /// Reserved 'avro' metadata key. /// From 00b32678263919748caa64a987cba5e9e2e9af84 Mon Sep 17 00:00:00 2001 From: Zoltan Csizmadia Date: Tue, 15 Feb 2022 20:15:29 -0600 Subject: [PATCH 13/28] Add interop-data tests --- lang/csharp/Avro.sln | 124 ++++++++++++++++++ lang/csharp/build.sh | 17 ++- .../Avro.Codec.Brotli.Test/BrotliTests.cs | 69 ---------- .../Avro.Codec.Brotli.csproj | 47 ------- .../apache/codec/Avro.Codec.Brotli/Brotli.cs | 109 --------------- .../Avro.Codec.Zstandard.Test.csproj | 42 ------ lang/csharp/src/apache/codec/Avro.Codec.sln | 101 -------------- .../Avro.File.BZip2.Test.csproj} | 2 +- .../BZip2Tests.cs | 2 +- .../Avro.File.BZip2.csproj} | 6 +- .../BZip2.cs | 11 +- .../Avro.File.Snappy.Test.csproj} | 2 +- .../SnappyTests.cs | 2 +- .../Avro.File.Snappy.csproj} | 6 +- .../Crc32.cs | 2 +- .../Snappy.cs | 10 +- .../Avro.File.XZ.Test.csproj} | 2 +- .../XZTests.cs | 8 +- .../Avro.File.XZ.csproj} | 6 +- .../{Avro.Codec.XZ => Avro.File.XZ}/XZ.cs | 20 +-- .../Avro.File.Zstandard.Test.csproj} | 2 +- .../ZstandardTests.cs | 2 +- .../Avro.File.Zstandard.csproj} | 6 +- .../Zstandard.cs | 9 +- lang/csharp/src/apache/main/File/Codec.cs | 6 +- lang/csharp/src/apache/test/Avro.test.csproj | 7 + .../test/Interop/InteropDataConstants.cs | 6 +- lang/csharp/versions.props | 6 + 28 files changed, 200 insertions(+), 432 deletions(-) delete mode 100644 lang/csharp/src/apache/codec/Avro.Codec.Brotli.Test/BrotliTests.cs delete mode 100644 lang/csharp/src/apache/codec/Avro.Codec.Brotli/Avro.Codec.Brotli.csproj delete mode 100644 lang/csharp/src/apache/codec/Avro.Codec.Brotli/Brotli.cs delete mode 100644 lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/Avro.Codec.Zstandard.Test.csproj delete mode 100644 lang/csharp/src/apache/codec/Avro.Codec.sln rename lang/csharp/src/apache/codec/{Avro.Codec.BZip2.Test/Avro.Codec.BZip2.Test.csproj => Avro.File.BZip2.Test/Avro.File.BZip2.Test.csproj} (95%) rename lang/csharp/src/apache/codec/{Avro.Codec.BZip2.Test => Avro.File.BZip2.Test}/BZip2Tests.cs (98%) rename lang/csharp/src/apache/codec/{Avro.Codec.BZip2/Avro.Codec.BZip2.csproj => Avro.File.BZip2/Avro.File.BZip2.csproj} (89%) rename lang/csharp/src/apache/codec/{Avro.Codec.BZip2 => Avro.File.BZip2}/BZip2.cs (91%) rename lang/csharp/src/apache/codec/{Avro.Codec.Brotli.Test/Avro.Codec.Brotli.Test.csproj => Avro.File.Snappy.Test/Avro.File.Snappy.Test.csproj} (95%) rename lang/csharp/src/apache/codec/{Avro.Codec.Snappy.Test => Avro.File.Snappy.Test}/SnappyTests.cs (98%) rename lang/csharp/src/apache/codec/{Avro.Codec.Snappy/Avro.Codec.Snappy.csproj => Avro.File.Snappy/Avro.File.Snappy.csproj} (90%) rename lang/csharp/src/apache/codec/{Avro.Codec.Snappy => Avro.File.Snappy}/Crc32.cs (98%) rename lang/csharp/src/apache/codec/{Avro.Codec.Snappy => Avro.File.Snappy}/Snappy.cs (94%) rename lang/csharp/src/apache/codec/{Avro.Codec.XZ.Test/Avro.Codec.XZ.Test.csproj => Avro.File.XZ.Test/Avro.File.XZ.Test.csproj} (95%) rename lang/csharp/src/apache/codec/{Avro.Codec.XZ.Test => Avro.File.XZ.Test}/XZTests.cs (94%) rename lang/csharp/src/apache/codec/{Avro.Codec.XZ/Avro.Codec.XZ.csproj => Avro.File.XZ/Avro.File.XZ.csproj} (93%) rename lang/csharp/src/apache/codec/{Avro.Codec.XZ => Avro.File.XZ}/XZ.cs (95%) rename lang/csharp/src/apache/codec/{Avro.Codec.Snappy.Test/Avro.Codec.Snappy.Test.csproj => Avro.File.Zstandard.Test/Avro.File.Zstandard.Test.csproj} (95%) rename lang/csharp/src/apache/codec/{Avro.Codec.Zstandard.Test => Avro.File.Zstandard.Test}/ZstandardTests.cs (98%) rename lang/csharp/src/apache/codec/{Avro.Codec.Zstandard/Avro.Codec.Zstandard.csproj => Avro.File.Zstandard/Avro.File.Zstandard.csproj} (90%) rename lang/csharp/src/apache/codec/{Avro.Codec.Zstandard => Avro.File.Zstandard}/Zstandard.cs (94%) diff --git a/lang/csharp/Avro.sln b/lang/csharp/Avro.sln index ec3d559be73..b945f4ead82 100644 --- a/lang/csharp/Avro.sln +++ b/lang/csharp/Avro.sln @@ -13,6 +13,24 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.msbuild", "src\apache\ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.perf", "src\apache\perf\Avro.perf.csproj", "{AC4E1909-2594-4D01-9B2B-B832C07BAFE5}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Codecs", "Codecs", "{0FAEE4F6-D72F-4B18-869A-7A90BAC1280F}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.BZip2", "src\apache\codec\Avro.File.BZip2\Avro.File.BZip2.csproj", "{FFA119B2-0D60-4090-B5A6-ECA718138812}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.BZip2.Test", "src\apache\codec\Avro.File.BZip2.Test\Avro.File.BZip2.Test.csproj", "{D5ED6642-3E33-493F-9217-FE00E4885699}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.Snappy", "src\apache\codec\Avro.File.Snappy\Avro.File.Snappy.csproj", "{B15BEEDC-A371-46D0-BFF6-63FC8105B520}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.Snappy.Test", "src\apache\codec\Avro.File.Snappy.Test\Avro.File.Snappy.Test.csproj", "{AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.XZ", "src\apache\codec\Avro.File.XZ\Avro.File.XZ.csproj", "{98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.XZ.Test", "src\apache\codec\Avro.File.XZ.Test\Avro.File.XZ.Test.csproj", "{99711F8E-C5C1-4864-A51F-3317E19CAD7B}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.Zstandard", "src\apache\codec\Avro.File.Zstandard\Avro.File.Zstandard.csproj", "{8207A628-6285-4DDF-B846-C0C7ED3E3D16}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.File.Zstandard.Test", "src\apache\codec\Avro.File.Zstandard.Test\Avro.File.Zstandard.Test.csproj", "{04264DDD-C204-4F59-88D4-FB4C69BD80C3}" +EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{8A671DF3-BC71-4E1A-BB06-0A225799A274}" ProjectSection(SolutionItems) = preProject ..\..\.editorconfig = ..\..\.editorconfig @@ -89,10 +107,116 @@ Global {AC4E1909-2594-4D01-9B2B-B832C07BAFE5}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU {AC4E1909-2594-4D01-9B2B-B832C07BAFE5}.Release|Mixed Platforms.Build.0 = Release|Any CPU {AC4E1909-2594-4D01-9B2B-B832C07BAFE5}.Release|x86.ActiveCfg = Release|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Debug|x86.ActiveCfg = Debug|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Debug|x86.Build.0 = Debug|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Release|Any CPU.Build.0 = Release|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Release|x86.ActiveCfg = Release|Any CPU + {FFA119B2-0D60-4090-B5A6-ECA718138812}.Release|x86.Build.0 = Release|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Debug|x86.ActiveCfg = Debug|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Debug|x86.Build.0 = Debug|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Release|Any CPU.Build.0 = Release|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Release|x86.ActiveCfg = Release|Any CPU + {D5ED6642-3E33-493F-9217-FE00E4885699}.Release|x86.Build.0 = Release|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Debug|x86.ActiveCfg = Debug|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Debug|x86.Build.0 = Debug|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Release|Any CPU.Build.0 = Release|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Release|x86.ActiveCfg = Release|Any CPU + {B15BEEDC-A371-46D0-BFF6-63FC8105B520}.Release|x86.Build.0 = Release|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Debug|x86.ActiveCfg = Debug|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Debug|x86.Build.0 = Debug|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Release|Any CPU.Build.0 = Release|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Release|x86.ActiveCfg = Release|Any CPU + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23}.Release|x86.Build.0 = Release|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Debug|x86.ActiveCfg = Debug|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Debug|x86.Build.0 = Debug|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Release|Any CPU.Build.0 = Release|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Release|x86.ActiveCfg = Release|Any CPU + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7}.Release|x86.Build.0 = Release|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Debug|x86.ActiveCfg = Debug|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Debug|x86.Build.0 = Debug|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Release|Any CPU.Build.0 = Release|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Release|x86.ActiveCfg = Release|Any CPU + {99711F8E-C5C1-4864-A51F-3317E19CAD7B}.Release|x86.Build.0 = Release|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Debug|x86.ActiveCfg = Debug|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Debug|x86.Build.0 = Debug|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Release|Any CPU.Build.0 = Release|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Release|x86.ActiveCfg = Release|Any CPU + {8207A628-6285-4DDF-B846-C0C7ED3E3D16}.Release|x86.Build.0 = Release|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Debug|x86.ActiveCfg = Debug|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Debug|x86.Build.0 = Debug|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Release|Any CPU.Build.0 = Release|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Release|x86.ActiveCfg = Release|Any CPU + {04264DDD-C204-4F59-88D4-FB4C69BD80C3}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {FFA119B2-0D60-4090-B5A6-ECA718138812} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F} + {D5ED6642-3E33-493F-9217-FE00E4885699} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F} + {B15BEEDC-A371-46D0-BFF6-63FC8105B520} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F} + {AA2CA9A3-71C0-4D16-B7E7-F6F50E400F23} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F} + {98CE721F-10AF-4665-9B14-3EA2CDF8F4C7} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F} + {99711F8E-C5C1-4864-A51F-3317E19CAD7B} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F} + {8207A628-6285-4DDF-B846-C0C7ED3E3D16} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F} + {04264DDD-C204-4F59-88D4-FB4C69BD80C3} = {0FAEE4F6-D72F-4B18-869A-7A90BAC1280F} + EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {ACE75CE8-16B2-4C6E-A5BE-B6F6DB5FE095} EndGlobalSection diff --git a/lang/csharp/build.sh b/lang/csharp/build.sh index f72baa54f9f..012bfe81f4b 100755 --- a/lang/csharp/build.sh +++ b/lang/csharp/build.sh @@ -38,10 +38,6 @@ do # AVRO-2442: Explicitly set LANG to work around ICU bug in `dotnet test` LANG=en_US.UTF-8 dotnet test --configuration Release --no-build \ --filter "TestCategory!=Interop" Avro.sln - - dotnet build --configuration Release src/apache/codec/Avro.Codec.sln - ANG=en_US.UTF-8 dotnet test --configuration Release --no-build \ - src/apache/codec/Avro.Codec.sln ;; perf) @@ -54,13 +50,20 @@ do dotnet pack --configuration Release Avro.sln # add the binary LICENSE and NOTICE to the tarball - mkdir build/ + mkdir -p build/ cp LICENSE NOTICE build/ # add binaries to the tarball - mkdir build/main/ + mkdir -p build/main/ cp -R src/apache/main/bin/Release/* build/main/ - mkdir build/codegen/ + # add codec binaries to the tarball + for codec in Avro.File.Snappy Avro.File.BZip2 Avro.File.XZ Avro.File.Zstandard + do + mkdir -p build/codec/$codec/ + cp -R src/apache/codec/$codec/bin/Release/* build/codec/$codec/ + done + # add codegen binaries to the tarball + mkdir -p build/codegen/ cp -R src/apache/codegen/bin/Release/* build/codegen/ # build the tarball diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Brotli.Test/BrotliTests.cs b/lang/csharp/src/apache/codec/Avro.Codec.Brotli.Test/BrotliTests.cs deleted file mode 100644 index e30653b8460..00000000000 --- a/lang/csharp/src/apache/codec/Avro.Codec.Brotli.Test/BrotliTests.cs +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -using System.IO; -using System.Linq; -using NUnit.Framework; - -namespace Avro.Codec.Brotli.Test -{ - public class Tests - { - private static int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 1 * 1024 * 1024 }; - - [Test, Combinatorial] - public void CompressDecompress([ValueSource(nameof(_testLengths))] int length, [Values] BrotliLevel level) - { - byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); - - BrotliCodec codec = new BrotliCodec(level); - - byte[] compressed = codec.Compress(data); - byte[] uncompressed = codec.Decompress(compressed, compressed.Length); - - CollectionAssert.AreEqual(data, uncompressed); - } - - [Test, Combinatorial] - public void CompressDecompressStream([ValueSource(nameof(_testLengths))] int length, [Values] BrotliLevel level) - { - byte[] data = Enumerable.Range(0, length).Select(x => (byte)x).ToArray(); - - BrotliCodec codec = new BrotliCodec(level); - - using (MemoryStream inputStream = new MemoryStream(data)) - using (MemoryStream outputStream = new MemoryStream()) - { - codec.Compress(inputStream, outputStream); - - byte[] compressed = outputStream.ToArray(); - byte[] uncompressed = codec.Decompress(compressed, compressed.Length); - - CollectionAssert.AreEqual(data, uncompressed); - } - } - - [Test] - public void ToStringAndName([Values] BrotliLevel level) - { - BrotliCodec codec = new BrotliCodec(level); - - Assert.AreEqual("brotli", codec.GetName()); - Assert.AreEqual($"brotli-{(int)level}", codec.ToString()); - } - } -} diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Brotli/Avro.Codec.Brotli.csproj b/lang/csharp/src/apache/codec/Avro.Codec.Brotli/Avro.Codec.Brotli.csproj deleted file mode 100644 index 05726d7d359..00000000000 --- a/lang/csharp/src/apache/codec/Avro.Codec.Brotli/Avro.Codec.Brotli.csproj +++ /dev/null @@ -1,47 +0,0 @@ - - - - - - - $(DefaultLibraryTargetFrameworks) - Avro.Codec.Brotli - true - ../../../../Avro.snk - - - - - Apache.Avro.Codec.Brotli - - - - - true - - - - - - - - - - - - diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Brotli/Brotli.cs b/lang/csharp/src/apache/codec/Avro.Codec.Brotli/Brotli.cs deleted file mode 100644 index 76c8947c575..00000000000 --- a/lang/csharp/src/apache/codec/Avro.Codec.Brotli/Brotli.cs +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -using System.IO; -using BrotliSharpLib; - -namespace Avro.Codec.Brotli -{ - /// - /// Brotli Compression level - /// - public enum BrotliLevel - { - Default = 1, - Level0 = 0, - Level1 = 1, - Level2 = 2, - Level3 = 3, - Level4 = 4, - Level5 = 5, - Level6 = 6, - Level7 = 7, - Level8 = 8, - Level9 = 9, - Level10 = 10, - Level11 = 11 - } - - /// - /// Implements Brotli compression and decompression. - /// - public class BrotliCodec : File.Codec - { - public const string DataFileConstant = "brotli"; - - private readonly BrotliLevel _level; - - public BrotliCodec() - : this(BrotliLevel.Default) - { - } - - public BrotliCodec(BrotliLevel level) - { - _level = level; - } - - /// - public override byte[] Compress(byte[] uncompressedData) - { - return BrotliSharpLib.Brotli.CompressBuffer(uncompressedData, 0, uncompressedData.Length, (int)_level); - } - - /// - public override void Compress(MemoryStream inputStream, MemoryStream outputStream) - { - outputStream.SetLength(0); - using (BrotliStream brotliStreams = new BrotliStream(outputStream, System.IO.Compression.CompressionMode.Compress, false)) - { - brotliStreams.SetQuality((int)_level); - inputStream.CopyTo(brotliStreams); - } - } - - /// - public override byte[] Decompress(byte[] compressedData, int blockLength) - { - return BrotliSharpLib.Brotli.DecompressBuffer(compressedData, 0, blockLength); - } - - /// - public override string GetName() - { - return DataFileConstant; - } - - /// - public override bool Equals(object other) - { - return this == other || GetType().Name == other.GetType().Name; - } - - /// - public override int GetHashCode() - { - return 0; - } - - /// - public override string ToString() - { - return $"{GetName()}-{(int)_level}"; - } - } -} diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/Avro.Codec.Zstandard.Test.csproj b/lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/Avro.Codec.Zstandard.Test.csproj deleted file mode 100644 index 0df9b50c342..00000000000 --- a/lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/Avro.Codec.Zstandard.Test.csproj +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - $(DefaultUnitTestTargetFrameworks) - false - - - - true - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/lang/csharp/src/apache/codec/Avro.Codec.sln b/lang/csharp/src/apache/codec/Avro.Codec.sln deleted file mode 100644 index 20e5afc268a..00000000000 --- a/lang/csharp/src/apache/codec/Avro.Codec.sln +++ /dev/null @@ -1,101 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Visual Studio Version 17 -VisualStudioVersion = 17.0.32126.317 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.Codec.BZip2", "Avro.Codec.BZip2\Avro.Codec.BZip2.csproj", "{40201607-AA97-44FB-8353-6CDFA803FBBB}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.Codec.BZip2.Test", "Avro.Codec.BZip2.Test\Avro.Codec.BZip2.Test.csproj", "{ABEDA4C6-8024-48C1-AFC0-37190ABC659A}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.Codec.Snappy", "Avro.Codec.Snappy\Avro.Codec.Snappy.csproj", "{C3F05185-2D65-4A63-B16D-6BE20A254C1F}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.Codec.Snappy.Test", "Avro.Codec.Snappy.Test\Avro.Codec.Snappy.Test.csproj", "{564A59B5-008D-4654-B63F-8EFAC8C45D10}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.Codec.XZ", "Avro.Codec.XZ\Avro.Codec.XZ.csproj", "{06396DF0-D656-4133-8BDC-EBF194D13AE4}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.Codec.XZ.Test", "Avro.Codec.XZ.Test\Avro.Codec.XZ.Test.csproj", "{4E8A6CCA-90DF-4350-B928-E6ADA5686883}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.Codec.Zstandard", "Avro.Codec.Zstandard\Avro.Codec.Zstandard.csproj", "{21E6C150-200F-484C-949F-C818BBE4F276}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.Codec.Zstandard.Test", "Avro.Codec.Zstandard.Test\Avro.Codec.Zstandard.Test.csproj", "{B93B1C26-C254-4C04-BB6B-E4FC8CCA9122}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.main", "..\main\Avro.main.csproj", "{E5B71984-FAEC-43E2-B2AC-4413D737C5CD}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Avro.Codec.Brotli", "Avro.Codec.Brotli\Avro.Codec.Brotli.csproj", "{9174B8A2-FBFF-4992-BF15-CE5729F695A7}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avro.Codec.Brotli.Test", "Avro.Codec.Brotli.Test\Avro.Codec.Brotli.Test.csproj", "{9CBB2B45-DEE4-432D-8841-F8524F1F4F9A}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {40201607-AA97-44FB-8353-6CDFA803FBBB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {40201607-AA97-44FB-8353-6CDFA803FBBB}.Debug|Any CPU.Build.0 = Debug|Any CPU - {40201607-AA97-44FB-8353-6CDFA803FBBB}.Release|Any CPU.ActiveCfg = Release|Any CPU - {40201607-AA97-44FB-8353-6CDFA803FBBB}.Release|Any CPU.Build.0 = Release|Any CPU - {ABEDA4C6-8024-48C1-AFC0-37190ABC659A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {ABEDA4C6-8024-48C1-AFC0-37190ABC659A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {ABEDA4C6-8024-48C1-AFC0-37190ABC659A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {ABEDA4C6-8024-48C1-AFC0-37190ABC659A}.Release|Any CPU.Build.0 = Release|Any CPU - {C3F05185-2D65-4A63-B16D-6BE20A254C1F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {C3F05185-2D65-4A63-B16D-6BE20A254C1F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {C3F05185-2D65-4A63-B16D-6BE20A254C1F}.Release|Any CPU.ActiveCfg = Release|Any CPU - {C3F05185-2D65-4A63-B16D-6BE20A254C1F}.Release|Any CPU.Build.0 = Release|Any CPU - {564A59B5-008D-4654-B63F-8EFAC8C45D10}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {564A59B5-008D-4654-B63F-8EFAC8C45D10}.Debug|Any CPU.Build.0 = Debug|Any CPU - {564A59B5-008D-4654-B63F-8EFAC8C45D10}.Release|Any CPU.ActiveCfg = Release|Any CPU - {564A59B5-008D-4654-B63F-8EFAC8C45D10}.Release|Any CPU.Build.0 = Release|Any CPU - {06396DF0-D656-4133-8BDC-EBF194D13AE4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {06396DF0-D656-4133-8BDC-EBF194D13AE4}.Debug|Any CPU.Build.0 = Debug|Any CPU - {06396DF0-D656-4133-8BDC-EBF194D13AE4}.Release|Any CPU.ActiveCfg = Release|Any CPU - {06396DF0-D656-4133-8BDC-EBF194D13AE4}.Release|Any CPU.Build.0 = Release|Any CPU - {4E8A6CCA-90DF-4350-B928-E6ADA5686883}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4E8A6CCA-90DF-4350-B928-E6ADA5686883}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4E8A6CCA-90DF-4350-B928-E6ADA5686883}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4E8A6CCA-90DF-4350-B928-E6ADA5686883}.Release|Any CPU.Build.0 = Release|Any CPU - {21E6C150-200F-484C-949F-C818BBE4F276}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {21E6C150-200F-484C-949F-C818BBE4F276}.Debug|Any CPU.Build.0 = Debug|Any CPU - {21E6C150-200F-484C-949F-C818BBE4F276}.Release|Any CPU.ActiveCfg = Release|Any CPU - {21E6C150-200F-484C-949F-C818BBE4F276}.Release|Any CPU.Build.0 = Release|Any CPU - {B93B1C26-C254-4C04-BB6B-E4FC8CCA9122}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {B93B1C26-C254-4C04-BB6B-E4FC8CCA9122}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B93B1C26-C254-4C04-BB6B-E4FC8CCA9122}.Release|Any CPU.ActiveCfg = Release|Any CPU - {B93B1C26-C254-4C04-BB6B-E4FC8CCA9122}.Release|Any CPU.Build.0 = Release|Any CPU - {E5B71984-FAEC-43E2-B2AC-4413D737C5CD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {E5B71984-FAEC-43E2-B2AC-4413D737C5CD}.Debug|Any CPU.Build.0 = Debug|Any CPU - {E5B71984-FAEC-43E2-B2AC-4413D737C5CD}.Release|Any CPU.ActiveCfg = Release|Any CPU - {E5B71984-FAEC-43E2-B2AC-4413D737C5CD}.Release|Any CPU.Build.0 = Release|Any CPU - {9174B8A2-FBFF-4992-BF15-CE5729F695A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {9174B8A2-FBFF-4992-BF15-CE5729F695A7}.Debug|Any CPU.Build.0 = Debug|Any CPU - {9174B8A2-FBFF-4992-BF15-CE5729F695A7}.Release|Any CPU.ActiveCfg = Release|Any CPU - {9174B8A2-FBFF-4992-BF15-CE5729F695A7}.Release|Any CPU.Build.0 = Release|Any CPU - {9CBB2B45-DEE4-432D-8841-F8524F1F4F9A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {9CBB2B45-DEE4-432D-8841-F8524F1F4F9A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {9CBB2B45-DEE4-432D-8841-F8524F1F4F9A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {9CBB2B45-DEE4-432D-8841-F8524F1F4F9A}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {32C95735-F692-433B-BCD2-A702CA701398} - EndGlobalSection -EndGlobal diff --git a/lang/csharp/src/apache/codec/Avro.Codec.BZip2.Test/Avro.Codec.BZip2.Test.csproj b/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/Avro.File.BZip2.Test.csproj similarity index 95% rename from lang/csharp/src/apache/codec/Avro.Codec.BZip2.Test/Avro.Codec.BZip2.Test.csproj rename to lang/csharp/src/apache/codec/Avro.File.BZip2.Test/Avro.File.BZip2.Test.csproj index 22d3b5d3d8a..ace1db23c2c 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.BZip2.Test/Avro.Codec.BZip2.Test.csproj +++ b/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/Avro.File.BZip2.Test.csproj @@ -29,7 +29,7 @@ - + diff --git a/lang/csharp/src/apache/codec/Avro.Codec.BZip2.Test/BZip2Tests.cs b/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/BZip2Tests.cs similarity index 98% rename from lang/csharp/src/apache/codec/Avro.Codec.BZip2.Test/BZip2Tests.cs rename to lang/csharp/src/apache/codec/Avro.File.BZip2.Test/BZip2Tests.cs index 62447a36704..cbb784ee92c 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.BZip2.Test/BZip2Tests.cs +++ b/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/BZip2Tests.cs @@ -19,7 +19,7 @@ using System.Linq; using NUnit.Framework; -namespace Avro.Codec.BZip2.Test +namespace Avro.File.BZip2.Test { public class Tests { diff --git a/lang/csharp/src/apache/codec/Avro.Codec.BZip2/Avro.Codec.BZip2.csproj b/lang/csharp/src/apache/codec/Avro.File.BZip2/Avro.File.BZip2.csproj similarity index 89% rename from lang/csharp/src/apache/codec/Avro.Codec.BZip2/Avro.Codec.BZip2.csproj rename to lang/csharp/src/apache/codec/Avro.File.BZip2/Avro.File.BZip2.csproj index e9123a55c91..d658c807871 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.BZip2/Avro.Codec.BZip2.csproj +++ b/lang/csharp/src/apache/codec/Avro.File.BZip2/Avro.File.BZip2.csproj @@ -20,14 +20,14 @@ $(DefaultLibraryTargetFrameworks) - Avro.Codec.BZip2 + Avro.File.BZip2 true ../../../../Avro.snk - Apache.Avro.Codec.BZip2 + Apache.Avro.File.BZip2 @@ -38,7 +38,7 @@ - + diff --git a/lang/csharp/src/apache/codec/Avro.Codec.BZip2/BZip2.cs b/lang/csharp/src/apache/codec/Avro.File.BZip2/BZip2.cs similarity index 91% rename from lang/csharp/src/apache/codec/Avro.Codec.BZip2/BZip2.cs rename to lang/csharp/src/apache/codec/Avro.File.BZip2/BZip2.cs index eabc01ee8fd..38c64fda68c 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.BZip2/BZip2.cs +++ b/lang/csharp/src/apache/codec/Avro.File.BZip2/BZip2.cs @@ -17,7 +17,7 @@ */ using System.IO; -namespace Avro.Codec.BZip2 +namespace Avro.File.BZip2 { /// /// BZip2 Compression level @@ -39,10 +39,8 @@ public enum BZip2Level /// /// Implements BZip2 compression and decompression. /// - public class BZip2Codec : File.Codec + public class BZip2Codec : Codec { - public const string DataFileConstant = "bzip2"; - private readonly BZip2Level _level; public BZip2Codec() @@ -61,7 +59,7 @@ public override byte[] Compress(byte[] uncompressedData) using (MemoryStream inputStream = new MemoryStream(uncompressedData)) using (MemoryStream outputStream = new MemoryStream()) { - ICSharpCode.SharpZipLib.BZip2.BZip2.Compress(inputStream, outputStream, false, (int)_level); + Compress(inputStream, outputStream); return outputStream.ToArray(); } } @@ -69,6 +67,7 @@ public override byte[] Compress(byte[] uncompressedData) /// public override void Compress(MemoryStream inputStream, MemoryStream outputStream) { + inputStream.Position = 0; outputStream.SetLength(0); ICSharpCode.SharpZipLib.BZip2.BZip2.Compress(inputStream, outputStream, false, (int)_level); } @@ -87,7 +86,7 @@ public override byte[] Decompress(byte[] compressedData, int blockLength) /// public override string GetName() { - return DataFileConstant; + return DataFileConstants.BZip2Codec; } /// diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Brotli.Test/Avro.Codec.Brotli.Test.csproj b/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/Avro.File.Snappy.Test.csproj similarity index 95% rename from lang/csharp/src/apache/codec/Avro.Codec.Brotli.Test/Avro.Codec.Brotli.Test.csproj rename to lang/csharp/src/apache/codec/Avro.File.Snappy.Test/Avro.File.Snappy.Test.csproj index 68e8e296c9e..ab325b0fa64 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Brotli.Test/Avro.Codec.Brotli.Test.csproj +++ b/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/Avro.File.Snappy.Test.csproj @@ -29,7 +29,7 @@ - + diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Snappy.Test/SnappyTests.cs b/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/SnappyTests.cs similarity index 98% rename from lang/csharp/src/apache/codec/Avro.Codec.Snappy.Test/SnappyTests.cs rename to lang/csharp/src/apache/codec/Avro.File.Snappy.Test/SnappyTests.cs index 7bf31ab258b..6c1255a27a2 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Snappy.Test/SnappyTests.cs +++ b/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/SnappyTests.cs @@ -19,7 +19,7 @@ using System.Linq; using NUnit.Framework; -namespace Avro.Codec.Snappy.Test +namespace Avro.File.Snappy.Test { public class Tests { diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Avro.Codec.Snappy.csproj b/lang/csharp/src/apache/codec/Avro.File.Snappy/Avro.File.Snappy.csproj similarity index 90% rename from lang/csharp/src/apache/codec/Avro.Codec.Snappy/Avro.Codec.Snappy.csproj rename to lang/csharp/src/apache/codec/Avro.File.Snappy/Avro.File.Snappy.csproj index 035f07f1ab7..2760d39b943 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Avro.Codec.Snappy.csproj +++ b/lang/csharp/src/apache/codec/Avro.File.Snappy/Avro.File.Snappy.csproj @@ -20,14 +20,14 @@ $(DefaultLibraryTargetFrameworks) - Avro.Codec.Snappy + Avro.File.Snappy true ../../../../Avro.snk - Apache.Avro.Codec.Snappy + Apache.Avro.File.Snappy @@ -38,7 +38,7 @@ - + diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Crc32.cs b/lang/csharp/src/apache/codec/Avro.File.Snappy/Crc32.cs similarity index 98% rename from lang/csharp/src/apache/codec/Avro.Codec.Snappy/Crc32.cs rename to lang/csharp/src/apache/codec/Avro.File.Snappy/Crc32.cs index c5fad481efe..1b832e11e43 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Crc32.cs +++ b/lang/csharp/src/apache/codec/Avro.File.Snappy/Crc32.cs @@ -17,7 +17,7 @@ */ using System; -namespace Avro.Codec.Snappy +namespace Avro.File.Snappy { /// /// Implements a 32-bit CRC hash algorithm. diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Snappy.cs b/lang/csharp/src/apache/codec/Avro.File.Snappy/Snappy.cs similarity index 94% rename from lang/csharp/src/apache/codec/Avro.Codec.Snappy/Snappy.cs rename to lang/csharp/src/apache/codec/Avro.File.Snappy/Snappy.cs index 5d815d9bf58..2237e5fbfd3 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Snappy/Snappy.cs +++ b/lang/csharp/src/apache/codec/Avro.File.Snappy/Snappy.cs @@ -18,15 +18,13 @@ using System; using System.IO; -namespace Avro.Codec.Snappy +namespace Avro.File.Snappy { /// /// Implements Snappy compression and decompression. /// - public class SnappyCodec : File.Codec + public class SnappyCodec : Codec { - public const string DataFileConstant = "snappy"; - /// /// public override byte[] Compress(byte[] uncompressedData) @@ -46,6 +44,8 @@ public override byte[] Compress(byte[] uncompressedData) /// public override void Compress(MemoryStream inputStream, MemoryStream outputStream) { + inputStream.Position = 0; + byte[] uncompressedData = inputStream.ToArray(); byte[] compressedData = IronSnappy.Snappy.Encode(uncompressedData); @@ -75,7 +75,7 @@ private static uint ByteSwap(uint word) /// public override string GetName() { - return DataFileConstant; + return DataFileConstants.SnappyCodec; } /// diff --git a/lang/csharp/src/apache/codec/Avro.Codec.XZ.Test/Avro.Codec.XZ.Test.csproj b/lang/csharp/src/apache/codec/Avro.File.XZ.Test/Avro.File.XZ.Test.csproj similarity index 95% rename from lang/csharp/src/apache/codec/Avro.Codec.XZ.Test/Avro.Codec.XZ.Test.csproj rename to lang/csharp/src/apache/codec/Avro.File.XZ.Test/Avro.File.XZ.Test.csproj index f400d1170f2..354c6a51c26 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.XZ.Test/Avro.Codec.XZ.Test.csproj +++ b/lang/csharp/src/apache/codec/Avro.File.XZ.Test/Avro.File.XZ.Test.csproj @@ -29,7 +29,7 @@ - + diff --git a/lang/csharp/src/apache/codec/Avro.Codec.XZ.Test/XZTests.cs b/lang/csharp/src/apache/codec/Avro.File.XZ.Test/XZTests.cs similarity index 94% rename from lang/csharp/src/apache/codec/Avro.Codec.XZ.Test/XZTests.cs rename to lang/csharp/src/apache/codec/Avro.File.XZ.Test/XZTests.cs index 46daecc57d5..05acd31aa67 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.XZ.Test/XZTests.cs +++ b/lang/csharp/src/apache/codec/Avro.File.XZ.Test/XZTests.cs @@ -19,18 +19,12 @@ using System.Linq; using NUnit.Framework; -namespace Avro.Codec.XZ.Test +namespace Avro.File.XZ.Test { public class Tests { private static int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 1 * 1024 * 1024 }; - [OneTimeSetUp] - public void OneTimeSetup() - { - XZCodec.Initialize(); - } - [Test, Combinatorial] public void CompressDecompress([ValueSource(nameof(_testLengths))] int length, [Values] XZLevel level) { diff --git a/lang/csharp/src/apache/codec/Avro.Codec.XZ/Avro.Codec.XZ.csproj b/lang/csharp/src/apache/codec/Avro.File.XZ/Avro.File.XZ.csproj similarity index 93% rename from lang/csharp/src/apache/codec/Avro.Codec.XZ/Avro.Codec.XZ.csproj rename to lang/csharp/src/apache/codec/Avro.File.XZ/Avro.File.XZ.csproj index d38a4a989ca..ee22d447eef 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.XZ/Avro.Codec.XZ.csproj +++ b/lang/csharp/src/apache/codec/Avro.File.XZ/Avro.File.XZ.csproj @@ -20,7 +20,7 @@ $(DefaultLibraryTargetFrameworks) - Avro.Codec.XZ + Avro.File.XZ true ../../../../Avro.snk CS8002 @@ -28,7 +28,7 @@ - Apache.Avro.Codec.XZ + Apache.Avro.File.XZ @@ -39,7 +39,7 @@ - + diff --git a/lang/csharp/src/apache/codec/Avro.Codec.XZ/XZ.cs b/lang/csharp/src/apache/codec/Avro.File.XZ/XZ.cs similarity index 95% rename from lang/csharp/src/apache/codec/Avro.Codec.XZ/XZ.cs rename to lang/csharp/src/apache/codec/Avro.File.XZ/XZ.cs index ae22c45ad67..572c0c409fe 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.XZ/XZ.cs +++ b/lang/csharp/src/apache/codec/Avro.File.XZ/XZ.cs @@ -21,7 +21,7 @@ using System.Runtime.InteropServices; using Joveler.Compression.XZ; -namespace Avro.Codec.XZ +namespace Avro.File.XZ { /// /// XZ Compression level @@ -46,10 +46,8 @@ public enum XZLevel /// /// Implements XZ compression and decompression. /// - public class XZCodec : File.Codec + public class XZCodec : Codec { - public const string DataFileConstant = "xz"; - private readonly XZLevel _level; private readonly bool _extreme; private readonly int _threads; @@ -76,11 +74,12 @@ public XZCodec(XZLevel level, bool extreme, int numOfThreads) _threads = numOfThreads; } - // !!! - // !!! This must be called as a global init before using this library - // !!! TODO: Make it more convinient - // !!! - public static void Initialize() + static XZCodec() + { + Initialize(); // One time initialization + } + + private static void Initialize() { string arch = RuntimeInformation.OSArchitecture.ToString().ToLower(); string foundLibPath = string.Empty; @@ -186,6 +185,7 @@ public override void Compress(MemoryStream inputStream, MemoryStream outputStrea Threads = _threads, }; + inputStream.Position = 0; outputStream.SetLength(0); using (XZStream xzStream = new XZStream(outputStream, compOpts, threadOpts)) @@ -213,7 +213,7 @@ public override byte[] Decompress(byte[] compressedData, int blockLength) /// public override string GetName() { - return DataFileConstant; + return DataFileConstants.XZCodec; } /// diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Snappy.Test/Avro.Codec.Snappy.Test.csproj b/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/Avro.File.Zstandard.Test.csproj similarity index 95% rename from lang/csharp/src/apache/codec/Avro.Codec.Snappy.Test/Avro.Codec.Snappy.Test.csproj rename to lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/Avro.File.Zstandard.Test.csproj index 2c399618015..651fabded68 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Snappy.Test/Avro.Codec.Snappy.Test.csproj +++ b/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/Avro.File.Zstandard.Test.csproj @@ -29,7 +29,7 @@ - + diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/ZstandardTests.cs b/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/ZstandardTests.cs similarity index 98% rename from lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/ZstandardTests.cs rename to lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/ZstandardTests.cs index 60d59033407..2344724ed59 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Zstandard.Test/ZstandardTests.cs +++ b/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/ZstandardTests.cs @@ -19,7 +19,7 @@ using System.Linq; using NUnit.Framework; -namespace Avro.Codec.Zstandard.Test +namespace Avro.File.Zstandard.Test { public class Tests { diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Avro.Codec.Zstandard.csproj b/lang/csharp/src/apache/codec/Avro.File.Zstandard/Avro.File.Zstandard.csproj similarity index 90% rename from lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Avro.Codec.Zstandard.csproj rename to lang/csharp/src/apache/codec/Avro.File.Zstandard/Avro.File.Zstandard.csproj index 4f912476620..e4471d72fca 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Avro.Codec.Zstandard.csproj +++ b/lang/csharp/src/apache/codec/Avro.File.Zstandard/Avro.File.Zstandard.csproj @@ -20,7 +20,7 @@ $(DefaultLibraryTargetFrameworks) - Avro.Codec.Zstandard + Avro.File.Zstandard true ../../../../Avro.snk CS8002 @@ -28,7 +28,7 @@ - Apache.Avro.Codec.Zstandard + Apache.Avro.File.Zstandard @@ -39,7 +39,7 @@ - + diff --git a/lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Zstandard.cs b/lang/csharp/src/apache/codec/Avro.File.Zstandard/Zstandard.cs similarity index 94% rename from lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Zstandard.cs rename to lang/csharp/src/apache/codec/Avro.File.Zstandard/Zstandard.cs index 2d49f4a6586..0230c696365 100644 --- a/lang/csharp/src/apache/codec/Avro.Codec.Zstandard/Zstandard.cs +++ b/lang/csharp/src/apache/codec/Avro.File.Zstandard/Zstandard.cs @@ -19,7 +19,7 @@ using System.IO; using ZstdNet; -namespace Avro.Codec.Zstandard +namespace Avro.File.Zstandard { /// /// Zstandard Compression level @@ -53,10 +53,8 @@ public enum ZstandardLevel /// /// Implements Zstandard compression and decompression. /// - public class ZstandardCodec : File.Codec + public class ZstandardCodec : Codec { - public const string DataFileConstant = "zstandard"; - private readonly ZstandardLevel _level; public ZstandardCodec() @@ -82,6 +80,7 @@ public override byte[] Compress(byte[] uncompressedData) /// public override void Compress(MemoryStream inputStream, MemoryStream outputStream) { + inputStream.Position = 0; byte[] compressedData = Compress(inputStream.ToArray()); outputStream.SetLength(0); @@ -100,7 +99,7 @@ public override byte[] Decompress(byte[] compressedData, int blockLength) /// public override string GetName() { - return DataFileConstant; + return DataFileConstants.ZstandardCodec; } /// diff --git a/lang/csharp/src/apache/main/File/Codec.cs b/lang/csharp/src/apache/main/File/Codec.cs index a36ac34a215..46191997a1d 100644 --- a/lang/csharp/src/apache/main/File/Codec.cs +++ b/lang/csharp/src/apache/main/File/Codec.cs @@ -166,9 +166,9 @@ public static Codec CreateCodec(Type codecType) case Type.XZ: case Type.Zstandard: { - // Create codec dynamically from "Avro.Codec.CODECNAME" assembly - Assembly assembly = Assembly.Load($"Avro.Codec.{codecType}"); - return assembly.CreateInstance($"Avro.Codec.{codecType}.{codecType}Codec") as Codec; + // Create codec dynamically from "Avro.File.CODECNAME" assembly + Assembly assembly = Assembly.Load($"Avro.File.{codecType}"); + return assembly.CreateInstance($"Avro.File.{codecType}.{codecType}Codec") as Codec; } } diff --git a/lang/csharp/src/apache/test/Avro.test.csproj b/lang/csharp/src/apache/test/Avro.test.csproj index a1864d89e04..6c359c81400 100644 --- a/lang/csharp/src/apache/test/Avro.test.csproj +++ b/lang/csharp/src/apache/test/Avro.test.csproj @@ -45,6 +45,13 @@ + + + + + + + diff --git a/lang/csharp/src/apache/test/Interop/InteropDataConstants.cs b/lang/csharp/src/apache/test/Interop/InteropDataConstants.cs index 94bfb408d18..170e28eb5fa 100644 --- a/lang/csharp/src/apache/test/Interop/InteropDataConstants.cs +++ b/lang/csharp/src/apache/test/Interop/InteropDataConstants.cs @@ -25,7 +25,11 @@ public class InteropDataConstants public static readonly HashSet SupportedCodecNames = new HashSet { DataFileConstants.NullCodec, - DataFileConstants.DeflateCodec + DataFileConstants.DeflateCodec, + DataFileConstants.SnappyCodec, + DataFileConstants.BZip2Codec, + DataFileConstants.XZCodec, + DataFileConstants.ZstandardCodec }; } } \ No newline at end of file diff --git a/lang/csharp/versions.props b/lang/csharp/versions.props index 2ec4d2e46ef..d53780c5494 100644 --- a/lang/csharp/versions.props +++ b/lang/csharp/versions.props @@ -31,6 +31,12 @@ 4.3.0 4.7.0 4.7.0 + + + 1.3.3 + 1.3.0 + 4.1.0 + 1.4.5 - + diff --git a/lang/csharp/src/apache/codec/Avro.File.Zstandard/Zstandard.cs b/lang/csharp/src/apache/codec/Avro.File.Zstandard/Zstandard.cs index 6eca1cb9985..b5d85a8f289 100644 --- a/lang/csharp/src/apache/codec/Avro.File.Zstandard/Zstandard.cs +++ b/lang/csharp/src/apache/codec/Avro.File.Zstandard/Zstandard.cs @@ -15,9 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -using System; using System.IO; -using ZstdNet; +using System.IO.Compression; +using Zstandard.Net; namespace Avro.File.Zstandard { @@ -70,10 +70,13 @@ public ZstandardCodec(ZstandardLevel level) /// public override byte[] Compress(byte[] uncompressedData) { - using (CompressionOptions options = new CompressionOptions((int)_level)) - using (Compressor compressor = new Compressor(options)) + using (var outputStream = new MemoryStream()) + using (var compressionStream = new ZstandardStream(outputStream, CompressionMode.Compress)) { - return compressor.Wrap(uncompressedData); + compressionStream.CompressionLevel = (int)_level; + compressionStream.Write(uncompressedData, 0, uncompressedData.Length); + compressionStream.Flush(); + return outputStream.ToArray(); } } @@ -81,18 +84,26 @@ public override byte[] Compress(byte[] uncompressedData) public override void Compress(MemoryStream inputStream, MemoryStream outputStream) { inputStream.Position = 0; - byte[] compressedData = Compress(inputStream.ToArray()); - outputStream.SetLength(0); - outputStream.Write(compressedData, 0, compressedData.Length); + + using (var compressionStream = new ZstandardStream(outputStream, CompressionMode.Compress, true)) + { + compressionStream.CompressionLevel = (int)_level; + inputStream.CopyTo(compressionStream); + compressionStream.Flush(); + } } /// public override byte[] Decompress(byte[] compressedData, int blockLength) { - using (Decompressor decompressor = new Decompressor()) + using (var memoryStream = new MemoryStream(compressedData, 0, blockLength)) + using (var outputStream = new MemoryStream()) + using (var compressionStream = new ZstandardStream(memoryStream, CompressionMode.Decompress)) { - return decompressor.Unwrap(compressedData); //.AsSpan(0, blockLength)); + compressionStream.CopyTo(outputStream); + compressionStream.Flush(); + return outputStream.ToArray(); } } diff --git a/lang/csharp/versions.props b/lang/csharp/versions.props index d53780c5494..6b767c60dd7 100644 --- a/lang/csharp/versions.props +++ b/lang/csharp/versions.props @@ -36,7 +36,7 @@ 1.3.3 1.3.0 4.1.0 - 1.4.5 + 1.1.7 Apache.Avro.File.BZip2 - + BZip2 compression library for Apache.Avro diff --git a/lang/csharp/src/apache/codec/Avro.File.Snappy/Avro.File.Snappy.csproj b/lang/csharp/src/apache/codec/Avro.File.Snappy/Avro.File.Snappy.csproj index 2760d39b943..71bc7968827 100644 --- a/lang/csharp/src/apache/codec/Avro.File.Snappy/Avro.File.Snappy.csproj +++ b/lang/csharp/src/apache/codec/Avro.File.Snappy/Avro.File.Snappy.csproj @@ -28,7 +28,7 @@ Apache.Avro.File.Snappy - + Snappy compression library for Apache.Avro diff --git a/lang/csharp/src/apache/codec/Avro.File.XZ/Avro.File.XZ.csproj b/lang/csharp/src/apache/codec/Avro.File.XZ/Avro.File.XZ.csproj index ee22d447eef..034bb99dcce 100644 --- a/lang/csharp/src/apache/codec/Avro.File.XZ/Avro.File.XZ.csproj +++ b/lang/csharp/src/apache/codec/Avro.File.XZ/Avro.File.XZ.csproj @@ -29,7 +29,7 @@ Apache.Avro.File.XZ - + XZ compression library for Apache.Avro diff --git a/lang/csharp/src/apache/codec/Avro.File.Zstandard/Avro.File.Zstandard.csproj b/lang/csharp/src/apache/codec/Avro.File.Zstandard/Avro.File.Zstandard.csproj index 28e72f93bd0..17f9f9f00f5 100644 --- a/lang/csharp/src/apache/codec/Avro.File.Zstandard/Avro.File.Zstandard.csproj +++ b/lang/csharp/src/apache/codec/Avro.File.Zstandard/Avro.File.Zstandard.csproj @@ -29,7 +29,7 @@ Apache.Avro.File.Zstandard - + Zstandard compression library for Apache.Avro From b094717fb2154ba0c42dc355eed04ca317657d95 Mon Sep 17 00:00:00 2001 From: Zoltan Csizmadia Date: Fri, 18 Feb 2022 06:50:19 -0600 Subject: [PATCH 26/28] Add codecs to README --- lang/csharp/README.md | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/lang/csharp/README.md b/lang/csharp/README.md index e2050c035a1..8d829c0b777 100644 --- a/lang/csharp/README.md +++ b/lang/csharp/README.md @@ -17,15 +17,19 @@ Install-Package Apache.Avro ## Project Target Frameworks -| Project | Type | .NET Standard 2.0 | .NET Standard 2.1 | .NET Core 3.1 | .NET 5.0 | .NET 6.0 | -|:---------------:|:----------:|:------------------:|:-----------------:|:-------------:|:---------:|:---------:| -| Avro.codegen | Exe | | | ✔️ |✔️ |✔️ | -| Avro.ipc | Library | ✔️ | ✔️ | | | | -| Avro.ipc.test | Unit Tests | | | ✔️ |✔️ |✔️ | -| Avro.main | Library | ✔️ | ✔️ | | | | -| Avro.msbuild | Library | ✔️ | ✔️ | | | | -| Avro.perf | Exe | | | ✔️ |✔️ |✔️ | -| Avro.test | Unit Tests | | | ✔️ |✔️ |✔️ | +| Project | Published to nuget.org | Type | .NET Standard 2.0 | .NET Standard 2.1 | .NET Core 3.1 | .NET 5.0 | .NET 6.0 | +|:-------------------:|:--------------------------:|:----------:|:------------------:|:-----------------:|:-------------:|:---------:|:---------:| +| Avro.main | Apache.Avro | Library | ✔️ | ✔️ | | | | +| Avro.File.Snappy | Apache.Avro.File.Snappy | Library | ✔️ | ✔️ | | | | +| Avro.File.BZip2 | Apache.Avro.File.BZip2 | Library | ✔️ | ✔️ | | | | +| Avro.File.XZ | Apache.Avro.File.XZ | Library | ✔️ | ✔️ | | | | +| Avro.File.Zstandard | Apache.Avro.File.Zstandard | Library | ✔️ | ✔️ | | | | +| Avro.codegen | Apache.Avro.Tools | Exe | | | ✔️ |✔️ |✔️ | +| Avro.ipc | | Library | ✔️ | ✔️ | | | | +| Avro.ipc.test | | Unit Tests | | | ✔️ |✔️ |✔️ | +| Avro.msbuild | | Library | ✔️ | ✔️ | | | | +| Avro.perf | | Exe | | | ✔️ |✔️ |✔️ | +| Avro.test | | Unit Tests | | | ✔️ |✔️ |✔️ | ## Dependency package version strategy From 71cd7319a202ec26c0f99aa0123bea03effbbabe Mon Sep 17 00:00:00 2001 From: Zoltan Csizmadia Date: Wed, 23 Feb 2022 08:42:49 -0600 Subject: [PATCH 27/28] Add readonly to member --- lang/csharp/src/apache/codec/Avro.File.BZip2.Test/BZip2Tests.cs | 2 +- .../src/apache/codec/Avro.File.Snappy.Test/SnappyTests.cs | 2 +- lang/csharp/src/apache/codec/Avro.File.XZ.Test/XZTests.cs | 2 +- .../src/apache/codec/Avro.File.Zstandard.Test/ZstandardTests.cs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/BZip2Tests.cs b/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/BZip2Tests.cs index f9fb5b8a4fd..83115378aa1 100644 --- a/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/BZip2Tests.cs +++ b/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/BZip2Tests.cs @@ -23,7 +23,7 @@ namespace Avro.File.BZip2.Test { public class Tests { - private static int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 100000 }; + private static readonly int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 100000 }; [Test, Combinatorial] public void CompressDecompress([ValueSource(nameof(_testLengths))] int length, [Values] BZip2Level level) diff --git a/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/SnappyTests.cs b/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/SnappyTests.cs index 2ea55bd2ea3..8845fca1616 100644 --- a/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/SnappyTests.cs +++ b/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/SnappyTests.cs @@ -23,7 +23,7 @@ namespace Avro.File.Snappy.Test { public class Tests { - private static int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 100000 }; + private static readonly int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 100000 }; [TestCaseSource(nameof(_testLengths))] public void CompressDecompress(int length) diff --git a/lang/csharp/src/apache/codec/Avro.File.XZ.Test/XZTests.cs b/lang/csharp/src/apache/codec/Avro.File.XZ.Test/XZTests.cs index cd53930c792..5a6205a5372 100644 --- a/lang/csharp/src/apache/codec/Avro.File.XZ.Test/XZTests.cs +++ b/lang/csharp/src/apache/codec/Avro.File.XZ.Test/XZTests.cs @@ -23,7 +23,7 @@ namespace Avro.File.XZ.Test { public class Tests { - private static int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 100000 }; + private static readonly int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 100000 }; [Test, Combinatorial] public void CompressDecompress([ValueSource(nameof(_testLengths))] int length, [Values] XZLevel level) diff --git a/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/ZstandardTests.cs b/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/ZstandardTests.cs index cf217869c04..10c82fc0d1f 100644 --- a/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/ZstandardTests.cs +++ b/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/ZstandardTests.cs @@ -23,7 +23,7 @@ namespace Avro.File.Zstandard.Test { public class Tests { - private static int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 100000 }; + private static readonly int[] _testLengths = new int[] { 0, 1000, 64 * 1024, 100000 }; [Test, Combinatorial] public void CompressDecompress([ValueSource(nameof(_testLengths))] int length, [Values] ZstandardLevel level) From 9725aba840cab111f9cb703fa146141c8c65590a Mon Sep 17 00:00:00 2001 From: Zoltan Csizmadia Date: Fri, 25 Feb 2022 13:02:37 -0600 Subject: [PATCH 28/28] Implement GetHashCode --- .../codec/Avro.File.BZip2.Test/BZip2Tests.cs | 28 +++++++++++++++++++ .../src/apache/codec/Avro.File.BZip2/BZip2.cs | 10 +++---- .../Avro.File.Snappy.Test/SnappyTests.cs | 20 +++++++++++++ .../apache/codec/Avro.File.Snappy/Snappy.cs | 2 +- .../apache/codec/Avro.File.XZ.Test/XZTests.cs | 28 +++++++++++++++++++ .../src/apache/codec/Avro.File.XZ/XZ.cs | 22 +++++++-------- .../ZstandardTests.cs | 28 +++++++++++++++++++ .../codec/Avro.File.Zstandard/Zstandard.cs | 12 ++++---- 8 files changed, 127 insertions(+), 23 deletions(-) diff --git a/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/BZip2Tests.cs b/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/BZip2Tests.cs index 83115378aa1..821cb4b7233 100644 --- a/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/BZip2Tests.cs +++ b/lang/csharp/src/apache/codec/Avro.File.BZip2.Test/BZip2Tests.cs @@ -65,5 +65,33 @@ public void ToStringAndName([Values] BZip2Level level) Assert.AreEqual("bzip2", codec.GetName()); Assert.AreEqual($"bzip2-{(int)level}", codec.ToString()); } + + [Test] + public void DefaultLevel() + { + BZip2Codec codec = new BZip2Codec(); + + Assert.AreEqual(BZip2Level.Default, codec.Level); + } + + [Test] + public void Equal([Values] BZip2Level level) + { + BZip2Codec codec1 = new BZip2Codec(level); + BZip2Codec codec2 = new BZip2Codec(level); + + Assert.IsTrue(codec1.Equals(codec1)); + Assert.IsTrue(codec2.Equals(codec2)); + Assert.IsTrue(codec1.Equals(codec2)); + Assert.IsTrue(codec2.Equals(codec1)); + } + + [Test] + public void HashCode([Values] BZip2Level level) + { + BZip2Codec codec = new BZip2Codec(level); + + Assert.AreNotEqual(0, codec.GetHashCode()); + } } } diff --git a/lang/csharp/src/apache/codec/Avro.File.BZip2/BZip2.cs b/lang/csharp/src/apache/codec/Avro.File.BZip2/BZip2.cs index f7d10617845..354c39cc89e 100644 --- a/lang/csharp/src/apache/codec/Avro.File.BZip2/BZip2.cs +++ b/lang/csharp/src/apache/codec/Avro.File.BZip2/BZip2.cs @@ -41,7 +41,7 @@ public enum BZip2Level /// public class BZip2Codec : Codec { - private readonly BZip2Level _level; + public BZip2Level Level {get; private set;} public BZip2Codec() : this(BZip2Level.Default) @@ -50,7 +50,7 @@ public BZip2Codec() public BZip2Codec(BZip2Level level) { - _level = level; + Level = level; } /// @@ -69,7 +69,7 @@ public override void Compress(MemoryStream inputStream, MemoryStream outputStrea { inputStream.Position = 0; outputStream.SetLength(0); - ICSharpCode.SharpZipLib.BZip2.BZip2.Compress(inputStream, outputStream, false, (int)_level); + ICSharpCode.SharpZipLib.BZip2.BZip2.Compress(inputStream, outputStream, false, (int)Level); } /// @@ -98,13 +98,13 @@ public override bool Equals(object other) /// public override int GetHashCode() { - return 0; + return GetName().GetHashCode(); } /// public override string ToString() { - return $"{GetName()}-{(int)_level}"; + return $"{GetName()}-{(int)Level}"; } } } diff --git a/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/SnappyTests.cs b/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/SnappyTests.cs index 8845fca1616..148d493aae2 100644 --- a/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/SnappyTests.cs +++ b/lang/csharp/src/apache/codec/Avro.File.Snappy.Test/SnappyTests.cs @@ -65,5 +65,25 @@ public void ToStringAndName() Assert.AreEqual("snappy", codec.GetName()); Assert.AreEqual("snappy", codec.ToString()); } + + [Test] + public void Equal() + { + SnappyCodec codec1 = new SnappyCodec(); + SnappyCodec codec2 = new SnappyCodec(); + + Assert.IsTrue(codec1.Equals(codec1)); + Assert.IsTrue(codec2.Equals(codec2)); + Assert.IsTrue(codec1.Equals(codec2)); + Assert.IsTrue(codec2.Equals(codec1)); + } + + [Test] + public void HashCode() + { + SnappyCodec codec = new SnappyCodec(); + + Assert.AreNotEqual(0, codec.GetHashCode()); + } } } diff --git a/lang/csharp/src/apache/codec/Avro.File.Snappy/Snappy.cs b/lang/csharp/src/apache/codec/Avro.File.Snappy/Snappy.cs index 2237e5fbfd3..a73f8209c0d 100644 --- a/lang/csharp/src/apache/codec/Avro.File.Snappy/Snappy.cs +++ b/lang/csharp/src/apache/codec/Avro.File.Snappy/Snappy.cs @@ -87,7 +87,7 @@ public override bool Equals(object other) /// public override int GetHashCode() { - return 0; + return GetName().GetHashCode(); } } } diff --git a/lang/csharp/src/apache/codec/Avro.File.XZ.Test/XZTests.cs b/lang/csharp/src/apache/codec/Avro.File.XZ.Test/XZTests.cs index 5a6205a5372..27f38dc2bf2 100644 --- a/lang/csharp/src/apache/codec/Avro.File.XZ.Test/XZTests.cs +++ b/lang/csharp/src/apache/codec/Avro.File.XZ.Test/XZTests.cs @@ -65,5 +65,33 @@ public void ToStringAndName([Values] XZLevel level) Assert.AreEqual("xz", codec.GetName()); Assert.AreEqual($"xz-{(int)level}", codec.ToString()); } + + [Test] + public void DefaultLevel() + { + XZCodec codec = new XZCodec(); + + Assert.AreEqual(XZLevel.Default, codec.Level); + } + + [Test] + public void Equal([Values] XZLevel level) + { + XZCodec codec1 = new XZCodec(level); + XZCodec codec2 = new XZCodec(level); + + Assert.IsTrue(codec1.Equals(codec1)); + Assert.IsTrue(codec2.Equals(codec2)); + Assert.IsTrue(codec1.Equals(codec2)); + Assert.IsTrue(codec2.Equals(codec1)); + } + + [Test] + public void HashCode([Values] XZLevel level) + { + XZCodec codec = new XZCodec(level); + + Assert.AreNotEqual(0, codec.GetHashCode()); + } } } diff --git a/lang/csharp/src/apache/codec/Avro.File.XZ/XZ.cs b/lang/csharp/src/apache/codec/Avro.File.XZ/XZ.cs index 1ab9b6bbd5d..84d3742f6cc 100644 --- a/lang/csharp/src/apache/codec/Avro.File.XZ/XZ.cs +++ b/lang/csharp/src/apache/codec/Avro.File.XZ/XZ.cs @@ -48,9 +48,9 @@ public enum XZLevel /// public class XZCodec : Codec { - private readonly XZLevel _level; - private readonly bool _extreme; - private readonly int _threads; + public XZLevel Level {get; private set;} + public bool Extreme {get; private set;} + public int Threads {get; private set;} public XZCodec() : this(XZLevel.Default) @@ -69,9 +69,9 @@ public XZCodec(XZLevel level, bool extreme) public XZCodec(XZLevel level, bool extreme, int numOfThreads) { - _level = level; - _extreme = extreme; - _threads = numOfThreads; + Level = level; + Extreme = extreme; + Threads = numOfThreads; } static XZCodec() @@ -175,14 +175,14 @@ public override void Compress(MemoryStream inputStream, MemoryStream outputStrea { XZCompressOptions compOpts = new XZCompressOptions { - Level = (LzmaCompLevel)(int)_level, - ExtremeFlag = _extreme, + Level = (LzmaCompLevel)(int)Level, + ExtremeFlag = Extreme, LeaveOpen = true }; XZThreadedCompressOptions threadOpts = new XZThreadedCompressOptions { - Threads = _threads, + Threads = Threads, }; inputStream.Position = 0; @@ -225,13 +225,13 @@ public override bool Equals(object other) /// public override int GetHashCode() { - return 0; + return GetName().GetHashCode(); } /// public override string ToString() { - return $"{GetName()}-{(int)_level}"; + return $"{GetName()}-{(int)Level}"; } } } diff --git a/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/ZstandardTests.cs b/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/ZstandardTests.cs index 10c82fc0d1f..e360ee576c8 100644 --- a/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/ZstandardTests.cs +++ b/lang/csharp/src/apache/codec/Avro.File.Zstandard.Test/ZstandardTests.cs @@ -65,5 +65,33 @@ public void ToStringAndName([Values] ZstandardLevel level) Assert.AreEqual("zstandard", codec.GetName()); Assert.AreEqual($"zstandard[{(int)level}]", codec.ToString()); } + + [Test] + public void DefaultLevel() + { + ZstandardCodec codec = new ZstandardCodec(); + + Assert.AreEqual(ZstandardLevel.Default, codec.Level); + } + + [Test] + public void Equal([Values] ZstandardLevel level) + { + ZstandardCodec codec1 = new ZstandardCodec(level); + ZstandardCodec codec2 = new ZstandardCodec(level); + + Assert.IsTrue(codec1.Equals(codec1)); + Assert.IsTrue(codec2.Equals(codec2)); + Assert.IsTrue(codec1.Equals(codec2)); + Assert.IsTrue(codec2.Equals(codec1)); + } + + [Test] + public void HashCode([Values] ZstandardLevel level) + { + ZstandardCodec codec = new ZstandardCodec(level); + + Assert.AreNotEqual(0, codec.GetHashCode()); + } } } diff --git a/lang/csharp/src/apache/codec/Avro.File.Zstandard/Zstandard.cs b/lang/csharp/src/apache/codec/Avro.File.Zstandard/Zstandard.cs index b5d85a8f289..5adfb441799 100644 --- a/lang/csharp/src/apache/codec/Avro.File.Zstandard/Zstandard.cs +++ b/lang/csharp/src/apache/codec/Avro.File.Zstandard/Zstandard.cs @@ -55,7 +55,7 @@ public enum ZstandardLevel /// public class ZstandardCodec : Codec { - private readonly ZstandardLevel _level; + public ZstandardLevel Level {get; private set;} public ZstandardCodec() : this(ZstandardLevel.Default) @@ -64,7 +64,7 @@ public ZstandardCodec() public ZstandardCodec(ZstandardLevel level) { - _level = level; + Level = level; } /// @@ -73,7 +73,7 @@ public override byte[] Compress(byte[] uncompressedData) using (var outputStream = new MemoryStream()) using (var compressionStream = new ZstandardStream(outputStream, CompressionMode.Compress)) { - compressionStream.CompressionLevel = (int)_level; + compressionStream.CompressionLevel = (int)Level; compressionStream.Write(uncompressedData, 0, uncompressedData.Length); compressionStream.Flush(); return outputStream.ToArray(); @@ -88,7 +88,7 @@ public override void Compress(MemoryStream inputStream, MemoryStream outputStrea using (var compressionStream = new ZstandardStream(outputStream, CompressionMode.Compress, true)) { - compressionStream.CompressionLevel = (int)_level; + compressionStream.CompressionLevel = (int)Level; inputStream.CopyTo(compressionStream); compressionStream.Flush(); } @@ -122,13 +122,13 @@ public override bool Equals(object other) /// public override int GetHashCode() { - return 0; + return GetName().GetHashCode(); } /// public override string ToString() { - return $"{GetName()}[{(int)_level}]"; + return $"{GetName()}[{(int)Level}]"; } } }