diff --git a/Jamfile b/Jamfile index b9acc0eb65..e4ab659753 100644 --- a/Jamfile +++ b/Jamfile @@ -23,7 +23,7 @@ for architectureObject in [ MultiArchSubDirSetup ] { # Include required packages: # primary architecture AddHaikuImagePackages [ FFilterByBuildFeatures - bash bc coreutils curl freetype icu libsolv zlib + bash bc coreutils curl freetype icu libsolv zlib zstd regular_image @{ bzip2 ctags diffutils expat ffmpeg findutils gawk glu grep gutenprint diff --git a/build/jam/BuildFeatures b/build/jam/BuildFeatures index fec1521fe0..28498c1d1d 100644 --- a/build/jam/BuildFeatures +++ b/build/jam/BuildFeatures @@ -677,5 +677,28 @@ if $(TARGET_PACKAGING_ARCH) = x86_64 { } } + +# zstd +if [ IsPackageAvailable zstd_devel ] { + ExtractBuildFeatureArchives zstd : + file: base zstd + runtime: lib + file: devel zstd_devel + depends: base + library: $(developLibDir)/libzstd.so + headers: $(developHeadersDir) + # sources are required for the primary architecture only + primary @{ + file: source zstd_source + sources: develop/sources/%portRevisionedName%/sources + }@ + ; + + EnableBuildFeatures zstd ; +} else { + Echo "zstd support not available on $(TARGET_PACKAGING_ARCH)" ; +} + + # ATA Drivers + Bus EnableBuildFeatures ata ; diff --git a/headers/build/private/support/ZstdCompressionAlgorithm.h b/headers/build/private/support/ZstdCompressionAlgorithm.h new file mode 100644 index 0000000000..a2f0a7c51c --- /dev/null +++ b/headers/build/private/support/ZstdCompressionAlgorithm.h @@ -0,0 +1 @@ +#include <../private/support/ZstdCompressionAlgorithm.h> diff --git a/headers/private/support/ZstdCompressionAlgorithm.h b/headers/private/support/ZstdCompressionAlgorithm.h new file mode 100644 index 0000000000..df75e1673d --- /dev/null +++ b/headers/private/support/ZstdCompressionAlgorithm.h @@ -0,0 +1,97 @@ +/* + * Copyright 2017, Jérôme Duval. + * Copyright 2014, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ +#ifndef _ZSTD_COMPRESSION_ALGORITHM_H_ +#define _ZSTD_COMPRESSION_ALGORITHM_H_ + + +#include + + +// compression level +enum { + B_ZSTD_COMPRESSION_NONE = 0, + B_ZSTD_COMPRESSION_FASTEST = 1, + B_ZSTD_COMPRESSION_BEST = 19, + B_ZSTD_COMPRESSION_DEFAULT = 2, +}; + + +class BZstdCompressionParameters : public BCompressionParameters { +public: + BZstdCompressionParameters( + int compressionLevel + = B_ZSTD_COMPRESSION_DEFAULT); + virtual ~BZstdCompressionParameters(); + + int32 CompressionLevel() const; + void SetCompressionLevel(int32 level); + + size_t BufferSize() const; + void SetBufferSize(size_t size); + +private: + int32 fCompressionLevel; + size_t fBufferSize; +}; + + +class BZstdDecompressionParameters : public BDecompressionParameters { +public: + BZstdDecompressionParameters(); + virtual ~BZstdDecompressionParameters(); + + size_t BufferSize() const; + void SetBufferSize(size_t size); + +private: + size_t fBufferSize; +}; + + +class BZstdCompressionAlgorithm : public BCompressionAlgorithm { +public: + BZstdCompressionAlgorithm(); + virtual ~BZstdCompressionAlgorithm(); + + virtual status_t CreateCompressingInputStream(BDataIO* input, + const BCompressionParameters* parameters, + BDataIO*& _stream); + virtual status_t CreateCompressingOutputStream(BDataIO* output, + const BCompressionParameters* parameters, + BDataIO*& _stream); + virtual status_t CreateDecompressingInputStream(BDataIO* input, + const BDecompressionParameters* parameters, + BDataIO*& _stream); + virtual status_t CreateDecompressingOutputStream(BDataIO* output, + const BDecompressionParameters* parameters, + BDataIO*& _stream); + + virtual status_t CompressBuffer(const void* input, + size_t inputSize, void* output, + size_t outputSize, size_t& _compressedSize, + const BCompressionParameters* parameters + = NULL); + virtual status_t DecompressBuffer(const void* input, + size_t inputSize, void* output, + size_t outputSize, + size_t& _uncompressedSize, + const BDecompressionParameters* parameters + = NULL); + +private: + struct CompressionStrategy; + struct DecompressionStrategy; + + template struct Stream; + template + friend struct Stream; + +private: + static status_t _TranslateZstdError(size_t error); +}; + + +#endif // _ZSTD_COMPRESSION_ALGORITHM_H_ diff --git a/src/kits/Jamfile b/src/kits/Jamfile index ca14ca89fc..cd091b3611 100644 --- a/src/kits/Jamfile +++ b/src/kits/Jamfile @@ -48,6 +48,7 @@ for architectureObject in [ MultiArchSubDirSetup ] { [ TargetLibstdc++ ] [ BuildFeatureAttribute icu : libraries ] [ BuildFeatureAttribute zlib : library ] + [ BuildFeatureAttribute zstd : library ] ; } } diff --git a/src/kits/support/Jamfile b/src/kits/support/Jamfile index 397e86d309..afc40985da 100644 --- a/src/kits/support/Jamfile +++ b/src/kits/support/Jamfile @@ -14,6 +14,14 @@ for architectureObject in [ MultiArchSubDirSetup ] { Includes [ FGristFiles ZlibCompressionAlgorithm.cpp ] : [ BuildFeatureAttribute zlib : headers ] ; + if [ FIsBuildFeatureEnabled zstd ] { + SubDirC++Flags -DZSTD_ENABLED ; + UseBuildFeatureHeaders zstd ; + Includes [ FGristFiles ZstdCompressionAlgorithm.cpp ] + : [ BuildFeatureAttribute zstd : headers ] ; + SetupFeatureObjectsDir zstd ; + } + # BUrl uses ICU to perform IDNA conversions (unicode domain names) UseBuildFeatureHeaders icu ; Includes [ FGristFiles Url.cpp ] @@ -46,6 +54,7 @@ for architectureObject in [ MultiArchSubDirSetup ] { Url.cpp Uuid.cpp ZlibCompressionAlgorithm.cpp + ZstdCompressionAlgorithm.cpp ; StaticLibrary [ MultiArchDefaultGristFiles libreferenceable.a ] diff --git a/src/kits/support/ZstdCompressionAlgorithm.cpp b/src/kits/support/ZstdCompressionAlgorithm.cpp new file mode 100644 index 0000000000..d6e0337f99 --- /dev/null +++ b/src/kits/support/ZstdCompressionAlgorithm.cpp @@ -0,0 +1,433 @@ +/* + * Copyright 2017, Jérôme Duval. + * Copyright 2014, Ingo Weinhold, ingo_weinhold@gmx.de. + * Distributed under the terms of the MIT License. + */ + + +#include + +#include +#include + +#include +#include + +#ifdef ZSTD_ENABLED + #include + #include +#endif + +#include + + +// build compression support only for userland +#if defined(ZSTD_ENABLED) && !defined(_KERNEL_MODE) && !defined(_BOOT_MODE) +# define B_ZSTD_COMPRESSION_SUPPORT 1 +#endif + + +static const size_t kMinBufferSize = 1024; +static const size_t kMaxBufferSize = 1024 * 1024; +static const size_t kDefaultBufferSize = 4 * 1024; + + +static size_t +sanitize_buffer_size(size_t size) +{ + if (size < kMinBufferSize) + return kMinBufferSize; + return std::min(size, kMaxBufferSize); +} + + +// #pragma mark - BZstdCompressionParameters + + +BZstdCompressionParameters::BZstdCompressionParameters( + int compressionLevel) + : + BCompressionParameters(), + fCompressionLevel(compressionLevel), + fBufferSize(kDefaultBufferSize) +{ +} + + +BZstdCompressionParameters::~BZstdCompressionParameters() +{ +} + + +int32 +BZstdCompressionParameters::CompressionLevel() const +{ + return fCompressionLevel; +} + + +void +BZstdCompressionParameters::SetCompressionLevel(int32 level) +{ + fCompressionLevel = level; +} + + +size_t +BZstdCompressionParameters::BufferSize() const +{ + return fBufferSize; +} + + +void +BZstdCompressionParameters::SetBufferSize(size_t size) +{ + fBufferSize = sanitize_buffer_size(size); +} + + +// #pragma mark - BZstdDecompressionParameters + + +BZstdDecompressionParameters::BZstdDecompressionParameters() + : + BDecompressionParameters(), + fBufferSize(kDefaultBufferSize) +{ +} + + +BZstdDecompressionParameters::~BZstdDecompressionParameters() +{ +} + + +size_t +BZstdDecompressionParameters::BufferSize() const +{ + return fBufferSize; +} + + +void +BZstdDecompressionParameters::SetBufferSize(size_t size) +{ + fBufferSize = sanitize_buffer_size(size); +} + + +// #pragma mark - CompressionStrategy + + +#ifdef B_ZSTD_COMPRESSION_SUPPORT + + +struct BZstdCompressionAlgorithm::CompressionStrategy { + typedef BZstdCompressionParameters Parameters; + + static const bool kNeedsFinalFlush = true; + + static size_t Init(ZSTD_CStream **stream, + const BZstdCompressionParameters* parameters) + { + int32 compressionLevel = B_ZSTD_COMPRESSION_DEFAULT; + if (parameters != NULL) { + compressionLevel = parameters->CompressionLevel(); + } + + *stream = ZSTD_createCStream(); + return ZSTD_initCStream(*stream, compressionLevel); + } + + static void Uninit(ZSTD_CStream *stream) + { + ZSTD_freeCStream(stream); + } + + static size_t Process(ZSTD_CStream *stream, ZSTD_inBuffer *input, + ZSTD_outBuffer *output, bool flush) + { + if (flush) + return ZSTD_flushStream(stream, output); + else + return ZSTD_compressStream(stream, output, input); + } +}; + + +#endif // B_ZSTD_COMPRESSION_SUPPORT + + +// #pragma mark - DecompressionStrategy + + +#ifdef ZSTD_ENABLED + + +struct BZstdCompressionAlgorithm::DecompressionStrategy { + typedef BZstdDecompressionParameters Parameters; + + static const bool kNeedsFinalFlush = false; + + static size_t Init(ZSTD_DStream **stream, + const BZstdDecompressionParameters* /*parameters*/) + { + *stream = ZSTD_createDStream(); + return ZSTD_initDStream(*stream); + } + + static void Uninit(ZSTD_DStream *stream) + { + ZSTD_freeDStream(stream); + } + + static size_t Process(ZSTD_DStream *stream, ZSTD_inBuffer *input, + ZSTD_outBuffer *output, bool flush) + { + return ZSTD_decompressStream(stream, output, input); + } + +}; + + +// #pragma mark - Stream + + +template +struct BZstdCompressionAlgorithm::Stream : BaseClass { + Stream(BDataIO* io) + : + BaseClass(io), + fStreamInitialized(false) + { + } + + ~Stream() + { + if (fStreamInitialized) { + if (Strategy::kNeedsFinalFlush) + this->Flush(); + Strategy::Uninit(fStream); + } + } + + status_t Init(const typename Strategy::Parameters* parameters) + { + status_t error = this->BaseClass::Init( + parameters != NULL ? parameters->BufferSize() : kDefaultBufferSize); + if (error != B_OK) + return error; + + size_t zstdError = Strategy::Init(&fStream, parameters); + if (ZSTD_getErrorCode(zstdError) != ZSTD_error_no_error) + return _TranslateZstdError(zstdError); + + fStreamInitialized = true; + return B_OK; + } + + virtual status_t ProcessData(const void* input, size_t inputSize, + void* output, size_t outputSize, size_t& bytesConsumed, + size_t& bytesProduced) + { + return _ProcessData(input, inputSize, output, outputSize, + bytesConsumed, bytesProduced, false); + } + + virtual status_t FlushPendingData(void* output, size_t outputSize, + size_t& bytesProduced) + { + size_t bytesConsumed; + return _ProcessData(NULL, 0, output, outputSize, + bytesConsumed, bytesProduced, true); + } + + template + static status_t Create(BDataIO* io, BaseParameters* _parameters, + BDataIO*& _stream) + { + const typename Strategy::Parameters* parameters +#ifdef _BOOT_MODE + = static_cast(_parameters); +#else + = dynamic_cast(_parameters); +#endif + Stream* stream = new(std::nothrow) Stream(io); + if (stream == NULL) + return B_NO_MEMORY; + + status_t error = stream->Init(parameters); + if (error != B_OK) { + delete stream; + return error; + } + + _stream = stream; + return B_OK; + } + +private: + status_t _ProcessData(const void* input, size_t inputSize, + void* output, size_t outputSize, size_t& bytesConsumed, + size_t& bytesProduced, bool flush) + { + inBuffer.src = input; + inBuffer.pos = 0; + inBuffer.size = inputSize; + outBuffer.dst = output; + outBuffer.pos = 0; + outBuffer.size = outputSize; + + size_t zstdError = Strategy::Process(fStream, &inBuffer, &outBuffer, flush); + if (ZSTD_getErrorCode(zstdError) != ZSTD_error_no_error) + return _TranslateZstdError(zstdError); + + bytesConsumed = inBuffer.pos; + bytesProduced = outBuffer.pos; + return B_OK; + } + +private: + bool fStreamInitialized; + StreamType *fStream; + ZSTD_inBuffer inBuffer; + ZSTD_outBuffer outBuffer; +}; + + +#endif // ZSTD_ENABLED + + +// #pragma mark - BZstdCompressionAlgorithm + + +BZstdCompressionAlgorithm::BZstdCompressionAlgorithm() + : + BCompressionAlgorithm() +{ +} + + +BZstdCompressionAlgorithm::~BZstdCompressionAlgorithm() +{ +} + + +status_t +BZstdCompressionAlgorithm::CreateCompressingInputStream(BDataIO* input, + const BCompressionParameters* parameters, BDataIO*& _stream) +{ +#ifdef B_ZSTD_COMPRESSION_SUPPORT + return Stream::Create( + input, parameters, _stream); +#else + return B_NOT_SUPPORTED; +#endif +} + + +status_t +BZstdCompressionAlgorithm::CreateCompressingOutputStream(BDataIO* output, + const BCompressionParameters* parameters, BDataIO*& _stream) +{ +#ifdef B_ZSTD_COMPRESSION_SUPPORT + return Stream::Create( + output, parameters, _stream); +#else + return B_NOT_SUPPORTED; +#endif +} + + +status_t +BZstdCompressionAlgorithm::CreateDecompressingInputStream(BDataIO* input, + const BDecompressionParameters* parameters, BDataIO*& _stream) +{ +#ifdef ZSTD_ENABLED + return Stream::Create( + input, parameters, _stream); +#else + return B_NOT_SUPPORTED; +#endif +} + + +status_t +BZstdCompressionAlgorithm::CreateDecompressingOutputStream(BDataIO* output, + const BDecompressionParameters* parameters, BDataIO*& _stream) +{ +#ifdef ZSTD_ENABLED + return Stream::Create( + output, parameters, _stream); +#else + return B_NOT_SUPPORTED; +#endif +} + + +status_t +BZstdCompressionAlgorithm::CompressBuffer(const void* input, + size_t inputSize, void* output, size_t outputSize, size_t& _compressedSize, + const BCompressionParameters* parameters) +{ +#ifdef B_ZSTD_COMPRESSION_SUPPORT + const BZstdCompressionParameters* zstdParameters + = dynamic_cast(parameters); + int compressionLevel = zstdParameters != NULL + ? zstdParameters->CompressionLevel() + : B_ZSTD_COMPRESSION_DEFAULT; + + size_t zstdError = ZSTD_compress(output, outputSize, input, + inputSize, compressionLevel); + if (ZSTD_isError(zstdError)) + return _TranslateZstdError(zstdError); + + _compressedSize = zstdError; + return B_OK; +#else + return B_NOT_SUPPORTED; +#endif +} + + +status_t +BZstdCompressionAlgorithm::DecompressBuffer(const void* input, + size_t inputSize, void* output, size_t outputSize, + size_t& _uncompressedSize, const BDecompressionParameters* parameters) +{ +#ifdef ZSTD_ENABLED + size_t zstdError = ZSTD_decompress(output, outputSize, input, + inputSize); + if (ZSTD_isError(zstdError)) + return _TranslateZstdError(zstdError); + + _uncompressedSize = zstdError; + return B_OK; +#else + return B_NOT_SUPPORTED; +#endif +} + + +/*static*/ status_t +BZstdCompressionAlgorithm::_TranslateZstdError(size_t error) +{ +#ifdef ZSTD_ENABLED + switch (ZSTD_getErrorCode(error)) { + case ZSTD_error_no_error: + return B_OK; + case ZSTD_error_seekableIO: + return B_BAD_VALUE; + case ZSTD_error_corruption_detected: + case ZSTD_error_checksum_wrong: + return B_BAD_DATA; + case ZSTD_error_version_unsupported: + return B_BAD_VALUE; + default: + return B_ERROR; + } +#else + return B_NOT_SUPPORTED; +#endif +} diff --git a/src/system/kernel/lib/Jamfile b/src/system/kernel/lib/Jamfile index 339945d020..a6d7054ac9 100644 --- a/src/system/kernel/lib/Jamfile +++ b/src/system/kernel/lib/Jamfile @@ -141,3 +141,6 @@ KernelMergeObject kernel_misc.o : HaikuSubInclude arch $(TARGET_ARCH) ; HaikuSubInclude zlib ; +if [ FIsBuildFeatureEnabled zstd ] { + HaikuSubInclude zstd ; +} diff --git a/src/system/kernel/lib/zstd/Jamfile b/src/system/kernel/lib/zstd/Jamfile new file mode 100644 index 0000000000..2d4a5d2950 --- /dev/null +++ b/src/system/kernel/lib/zstd/Jamfile @@ -0,0 +1,26 @@ +SubDir HAIKU_TOP src system kernel lib zstd ; + +local zstdSourceDirectory = [ BuildFeatureAttribute zstd : sources : path ] ; +UseHeaders [ FDirName $(zstdSourceDirectory) lib ] ; +UseHeaders [ FDirName $(zstdSourceDirectory) lib common ] ; + +local zstdCommonSources = + error_private.c + entropy_common.c fse_decompress.c zstd_common.c + xxhash.c + ; +local zstdDecSources = + huf_decompress.c zstd_decompress.c + ; + +LOCATE on [ FGristFiles $(zstdCommonSources) ] = + [ FDirName $(zstdSourceDirectory) lib common ] ; +LOCATE on [ FGristFiles $(zstdDecSources) ] = + [ FDirName $(zstdSourceDirectory) lib decompress ] ; +Depends [ FGristFiles $(zstdCommonSources) $(zstdDecSources) ] + : [ BuildFeatureAttribute zstd : sources ] ; + +# Build zstd with PIC, such that it can be used by kernel add-ons (filesystems). +KernelStaticLibrary kernel_libzstd.a : + $(zstdCommonSources) $(zstdDecSources) + ; diff --git a/src/tests/kits/support/compression_test.cpp b/src/tests/kits/support/compression_test.cpp index c752a90e6d..3a0d205453 100644 --- a/src/tests/kits/support/compression_test.cpp +++ b/src/tests/kits/support/compression_test.cpp @@ -13,6 +13,7 @@ #include #include +#include extern const char* __progname; @@ -22,6 +23,7 @@ const char* kCommandName = __progname; enum CompressionType { ZlibCompression, GzipCompression, + ZstdCompression, }; @@ -36,7 +38,8 @@ static const char* kUsage = " -d, --decompress\n" " Decompress the input file (default is compress).\n" " -f \n" - " Specify the compression format: \"zlib\" (default), or \"gzip\"\n" + " Specify the compression format: \"zlib\" (default), \"gzip\"\n" + " or \"zstd\".\n" " -h, --help\n" " Print this usage info.\n" " -i, --input-stream\n" @@ -55,9 +58,9 @@ print_usage_and_exit(bool error) int main(int argc, const char* const* argv) { - int compressionLevel = B_ZLIB_COMPRESSION_DEFAULT; + int compressionLevel = -1; bool compress = true; - bool useInputStream = true; + bool useInputStream = false; CompressionType compressionType = ZlibCompression; while (true) { @@ -101,6 +104,8 @@ main(int argc, const char* const* argv) compressionType = ZlibCompression; } else if (strcmp(optarg, "gzip") == 0) { compressionType = GzipCompression; + } else if (strcmp(optarg, "zstd") == 0) { + compressionType = ZstdCompression; } else { fprintf(stderr, "Error: Unsupported compression type " "\"%s\"\n", optarg); @@ -152,6 +157,8 @@ main(int argc, const char* const* argv) case ZlibCompression: case GzipCompression: { + if (compressionLevel < 0) + compressionLevel = B_ZLIB_COMPRESSION_DEFAULT; compressionAlgorithm = new BZlibCompressionAlgorithm; BZlibCompressionParameters* zlibCompressionParameters = new BZlibCompressionParameters(compressionLevel); @@ -161,6 +168,16 @@ main(int argc, const char* const* argv) decompressionParameters = new BZlibDecompressionParameters; break; } + case ZstdCompression: + { + if (compressionLevel < 0) + compressionLevel = B_ZSTD_COMPRESSION_DEFAULT; + compressionAlgorithm = new BZstdCompressionAlgorithm; + compressionParameters + = new BZstdCompressionParameters(compressionLevel); + decompressionParameters = new BZstdDecompressionParameters; + break; + } } if (useInputStream) {