Add ZstdCompressionAlgorithm.

* kernel lib is decompress only (like zlib).
* expand compression_test for zstd.

Jamfile

@@ -23,7 +23,7 @@ for architectureObject in [ MultiArchSubDirSetup ] {
 # Include required packages:
 # primary architecture
 AddHaikuImagePackages [ FFilterByBuildFeatures
-	bash bc coreutils curl freetype icu libsolv zlib
+	bash bc coreutils curl freetype icu libsolv zlib zstd
 
 	regular_image @{
 		bzip2 ctags diffutils expat ffmpeg findutils gawk glu grep gutenprint

build/jam/BuildFeatures

@@ -677,5 +677,28 @@ if $(TARGET_PACKAGING_ARCH) = x86_64 {
 	}
 }
 
+
+# zstd
+if [ IsPackageAvailable zstd_devel ] {
+	ExtractBuildFeatureArchives zstd :
+		file: base zstd
+			runtime: lib
+		file: devel zstd_devel
+			depends: base
+			library: $(developLibDir)/libzstd.so
+			headers: $(developHeadersDir)
+		# sources are required for the primary architecture only
+		primary @{
+			file: source zstd_source
+				sources: develop/sources/%portRevisionedName%/sources
+		}@
+		;
+
+	EnableBuildFeatures zstd ;
+} else {
+	Echo "zstd support not available on $(TARGET_PACKAGING_ARCH)" ;
+}
+
+
 # ATA Drivers + Bus
 EnableBuildFeatures ata ;

headers/build/private/support/ZstdCompressionAlgorithm.h

@@ -0,0 +1 @@
+#include <../private/support/ZstdCompressionAlgorithm.h>

headers/private/support/ZstdCompressionAlgorithm.h

@@ -0,0 +1,97 @@
+/*
+ * Copyright 2017, Jérôme Duval.
+ * Copyright 2014, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef _ZSTD_COMPRESSION_ALGORITHM_H_
+#define _ZSTD_COMPRESSION_ALGORITHM_H_
+
+
+#include <CompressionAlgorithm.h>
+
+
+// compression level
+enum {
+	B_ZSTD_COMPRESSION_NONE		= 0,
+	B_ZSTD_COMPRESSION_FASTEST	= 1,
+	B_ZSTD_COMPRESSION_BEST		= 19,
+	B_ZSTD_COMPRESSION_DEFAULT	= 2,
+};
+
+
+class BZstdCompressionParameters : public BCompressionParameters {
+public:
+								BZstdCompressionParameters(
+									int compressionLevel
+										= B_ZSTD_COMPRESSION_DEFAULT);
+	virtual						~BZstdCompressionParameters();
+
+			int32				CompressionLevel() const;
+			void				SetCompressionLevel(int32 level);
+
+			size_t				BufferSize() const;
+			void				SetBufferSize(size_t size);
+
+private:
+			int32				fCompressionLevel;
+			size_t				fBufferSize;
+};
+
+
+class BZstdDecompressionParameters : public BDecompressionParameters {
+public:
+								BZstdDecompressionParameters();
+	virtual						~BZstdDecompressionParameters();
+
+			size_t				BufferSize() const;
+			void				SetBufferSize(size_t size);
+
+private:
+			size_t				fBufferSize;
+};
+
+
+class BZstdCompressionAlgorithm : public BCompressionAlgorithm {
+public:
+								BZstdCompressionAlgorithm();
+	virtual						~BZstdCompressionAlgorithm();
+
+	virtual	status_t			CreateCompressingInputStream(BDataIO* input,
+									const BCompressionParameters* parameters,
+									BDataIO*& _stream);
+	virtual	status_t			CreateCompressingOutputStream(BDataIO* output,
+									const BCompressionParameters* parameters,
+									BDataIO*& _stream);
+	virtual	status_t			CreateDecompressingInputStream(BDataIO* input,
+									const BDecompressionParameters* parameters,
+									BDataIO*& _stream);
+	virtual	status_t			CreateDecompressingOutputStream(BDataIO* output,
+									const BDecompressionParameters* parameters,
+									BDataIO*& _stream);
+
+	virtual	status_t			CompressBuffer(const void* input,
+									size_t inputSize, void* output,
+									size_t outputSize, size_t& _compressedSize,
+									const BCompressionParameters* parameters
+										= NULL);
+	virtual	status_t			DecompressBuffer(const void* input,
+									size_t inputSize, void* output,
+									size_t outputSize,
+									size_t& _uncompressedSize,
+									const BDecompressionParameters* parameters
+										= NULL);
+
+private:
+			struct CompressionStrategy;
+			struct DecompressionStrategy;
+
+			template<typename BaseClass, typename Strategy, typename StreamType> struct Stream;
+			template<typename BaseClass, typename Strategy, typename StreamType>
+				friend struct Stream;
+
+private:
+	static	status_t			_TranslateZstdError(size_t error);
+};
+
+
+#endif	// _ZSTD_COMPRESSION_ALGORITHM_H_

src/kits/Jamfile

@@ -48,6 +48,7 @@ for architectureObject in [ MultiArchSubDirSetup ] {
 			[ TargetLibstdc++ ]
 			[ BuildFeatureAttribute icu : libraries ]
 			[ BuildFeatureAttribute zlib : library ]
+			[ BuildFeatureAttribute zstd : library ]
 			;
 	}
 }

src/kits/support/Jamfile

@@ -14,6 +14,14 @@ for architectureObject in [ MultiArchSubDirSetup ] {
 		Includes [ FGristFiles ZlibCompressionAlgorithm.cpp ]
 			: [ BuildFeatureAttribute zlib : headers ] ;
 
+		if [ FIsBuildFeatureEnabled zstd ] {
+			SubDirC++Flags -DZSTD_ENABLED ;
+			UseBuildFeatureHeaders zstd ;
+			Includes [ FGristFiles ZstdCompressionAlgorithm.cpp ]
+				: [ BuildFeatureAttribute zstd : headers ] ;
+			SetupFeatureObjectsDir zstd ;
+		}
+
 		# BUrl uses ICU to perform IDNA conversions (unicode domain names)
 		UseBuildFeatureHeaders icu ;
 		Includes [ FGristFiles Url.cpp ]
@@ -46,6 +54,7 @@ for architectureObject in [ MultiArchSubDirSetup ] {
 			Url.cpp
 			Uuid.cpp
 			ZlibCompressionAlgorithm.cpp
+			ZstdCompressionAlgorithm.cpp
 			;
 
 		StaticLibrary [ MultiArchDefaultGristFiles libreferenceable.a ]

src/kits/support/ZstdCompressionAlgorithm.cpp

@@ -0,0 +1,433 @@
+/*
+ * Copyright 2017, Jérôme Duval.
+ * Copyright 2014, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Distributed under the terms of the MIT License.
+ */
+
+
+#include <ZstdCompressionAlgorithm.h>
+
+#include <errno.h>
+#include <string.h>
+
+#include <algorithm>
+#include <new>
+
+#ifdef ZSTD_ENABLED
+  #include <zstd.h>
+  #include <zstd_errors.h>
+#endif
+
+#include <DataIO.h>
+
+
+// build compression support only for userland
+#if defined(ZSTD_ENABLED) && !defined(_KERNEL_MODE) && !defined(_BOOT_MODE)
+#	define B_ZSTD_COMPRESSION_SUPPORT 1
+#endif
+
+
+static const size_t kMinBufferSize		= 1024;
+static const size_t kMaxBufferSize		= 1024 * 1024;
+static const size_t kDefaultBufferSize	= 4 * 1024;
+
+
+static size_t
+sanitize_buffer_size(size_t size)
+{
+	if (size < kMinBufferSize)
+		return kMinBufferSize;
+	return std::min(size, kMaxBufferSize);
+}
+
+
+// #pragma mark - BZstdCompressionParameters
+
+
+BZstdCompressionParameters::BZstdCompressionParameters(
+	int compressionLevel)
+	:
+	BCompressionParameters(),
+	fCompressionLevel(compressionLevel),
+	fBufferSize(kDefaultBufferSize)
+{
+}
+
+
+BZstdCompressionParameters::~BZstdCompressionParameters()
+{
+}
+
+
+int32
+BZstdCompressionParameters::CompressionLevel() const
+{
+	return fCompressionLevel;
+}
+
+
+void
+BZstdCompressionParameters::SetCompressionLevel(int32 level)
+{
+	fCompressionLevel = level;
+}
+
+
+size_t
+BZstdCompressionParameters::BufferSize() const
+{
+	return fBufferSize;
+}
+
+
+void
+BZstdCompressionParameters::SetBufferSize(size_t size)
+{
+	fBufferSize = sanitize_buffer_size(size);
+}
+
+
+// #pragma mark - BZstdDecompressionParameters
+
+
+BZstdDecompressionParameters::BZstdDecompressionParameters()
+	:
+	BDecompressionParameters(),
+	fBufferSize(kDefaultBufferSize)
+{
+}
+
+
+BZstdDecompressionParameters::~BZstdDecompressionParameters()
+{
+}
+
+
+size_t
+BZstdDecompressionParameters::BufferSize() const
+{
+	return fBufferSize;
+}
+
+
+void
+BZstdDecompressionParameters::SetBufferSize(size_t size)
+{
+	fBufferSize = sanitize_buffer_size(size);
+}
+
+
+// #pragma mark - CompressionStrategy
+
+
+#ifdef B_ZSTD_COMPRESSION_SUPPORT
+
+
+struct BZstdCompressionAlgorithm::CompressionStrategy {
+	typedef BZstdCompressionParameters Parameters;
+
+	static const bool kNeedsFinalFlush = true;
+
+	static size_t Init(ZSTD_CStream **stream,
+		const BZstdCompressionParameters* parameters)
+	{
+		int32 compressionLevel = B_ZSTD_COMPRESSION_DEFAULT;
+		if (parameters != NULL) {
+			compressionLevel = parameters->CompressionLevel();
+		}
+
+		*stream = ZSTD_createCStream();
+		return ZSTD_initCStream(*stream, compressionLevel);
+	}
+
+	static void Uninit(ZSTD_CStream *stream)
+	{
+		ZSTD_freeCStream(stream);
+	}
+
+	static size_t Process(ZSTD_CStream *stream, ZSTD_inBuffer *input,
+		ZSTD_outBuffer *output, bool flush)
+	{
+		if (flush)
+			return ZSTD_flushStream(stream, output);
+		else
+			return ZSTD_compressStream(stream, output, input);
+	}
+};
+
+
+#endif	// B_ZSTD_COMPRESSION_SUPPORT
+
+
+// #pragma mark - DecompressionStrategy
+
+
+#ifdef ZSTD_ENABLED
+
+
+struct BZstdCompressionAlgorithm::DecompressionStrategy {
+	typedef BZstdDecompressionParameters Parameters;
+
+	static const bool kNeedsFinalFlush = false;
+
+	static size_t Init(ZSTD_DStream **stream,
+		const BZstdDecompressionParameters* /*parameters*/)
+	{
+		*stream = ZSTD_createDStream();
+		return ZSTD_initDStream(*stream);
+	}
+
+	static void Uninit(ZSTD_DStream *stream)
+	{
+		ZSTD_freeDStream(stream);
+	}
+
+	static size_t Process(ZSTD_DStream *stream, ZSTD_inBuffer *input,
+		ZSTD_outBuffer *output, bool flush)
+	{
+		return ZSTD_decompressStream(stream, output, input);
+	}
+
+};
+
+
+// #pragma mark - Stream
+
+
+template<typename BaseClass, typename Strategy, typename StreamType>
+struct BZstdCompressionAlgorithm::Stream : BaseClass {
+	Stream(BDataIO* io)
+		:
+		BaseClass(io),
+		fStreamInitialized(false)
+	{
+	}
+
+	~Stream()
+	{
+		if (fStreamInitialized) {
+			if (Strategy::kNeedsFinalFlush)
+				this->Flush();
+			Strategy::Uninit(fStream);
+		}
+	}
+
+	status_t Init(const typename Strategy::Parameters* parameters)
+	{
+		status_t error = this->BaseClass::Init(
+			parameters != NULL ? parameters->BufferSize() : kDefaultBufferSize);
+		if (error != B_OK)
+			return error;
+
+		size_t zstdError = Strategy::Init(&fStream, parameters);
+		if (ZSTD_getErrorCode(zstdError) != ZSTD_error_no_error)
+			return _TranslateZstdError(zstdError);
+
+		fStreamInitialized = true;
+		return B_OK;
+	}
+
+	virtual status_t ProcessData(const void* input, size_t inputSize,
+		void* output, size_t outputSize, size_t& bytesConsumed,
+		size_t& bytesProduced)
+	{
+		return _ProcessData(input, inputSize, output, outputSize,
+			bytesConsumed, bytesProduced, false);
+	}
+
+	virtual status_t FlushPendingData(void* output, size_t outputSize,
+		size_t& bytesProduced)
+	{
+		size_t bytesConsumed;
+		return _ProcessData(NULL, 0, output, outputSize,
+			bytesConsumed, bytesProduced, true);
+	}
+
+	template<typename BaseParameters>
+	static status_t Create(BDataIO* io, BaseParameters* _parameters,
+		BDataIO*& _stream)
+	{
+		const typename Strategy::Parameters* parameters
+#ifdef _BOOT_MODE
+			= static_cast<const typename Strategy::Parameters*>(_parameters);
+#else
+			= dynamic_cast<const typename Strategy::Parameters*>(_parameters);
+#endif
+		Stream* stream = new(std::nothrow) Stream(io);
+		if (stream == NULL)
+			return B_NO_MEMORY;
+
+		status_t error = stream->Init(parameters);
+		if (error != B_OK) {
+			delete stream;
+			return error;
+		}
+
+		_stream = stream;
+		return B_OK;
+	}
+
+private:
+	status_t _ProcessData(const void* input, size_t inputSize,
+		void* output, size_t outputSize, size_t& bytesConsumed,
+		size_t& bytesProduced, bool flush)
+	{
+		inBuffer.src = input;
+		inBuffer.pos = 0;
+		inBuffer.size = inputSize;
+		outBuffer.dst = output;
+		outBuffer.pos = 0;
+		outBuffer.size = outputSize;
+
+		size_t zstdError = Strategy::Process(fStream, &inBuffer, &outBuffer, flush);
+		if (ZSTD_getErrorCode(zstdError) != ZSTD_error_no_error)
+			return _TranslateZstdError(zstdError);
+
+		bytesConsumed = inBuffer.pos;
+		bytesProduced = outBuffer.pos;
+		return B_OK;
+	}
+
+private:
+	bool		fStreamInitialized;
+	StreamType	*fStream;
+	ZSTD_inBuffer inBuffer;
+	ZSTD_outBuffer outBuffer;
+};
+
+
+#endif	// ZSTD_ENABLED
+
+
+// #pragma mark - BZstdCompressionAlgorithm
+
+
+BZstdCompressionAlgorithm::BZstdCompressionAlgorithm()
+	:
+	BCompressionAlgorithm()
+{
+}
+
+
+BZstdCompressionAlgorithm::~BZstdCompressionAlgorithm()
+{
+}
+
+
+status_t
+BZstdCompressionAlgorithm::CreateCompressingInputStream(BDataIO* input,
+	const BCompressionParameters* parameters, BDataIO*& _stream)
+{
+#ifdef B_ZSTD_COMPRESSION_SUPPORT
+	return Stream<BAbstractInputStream, CompressionStrategy, ZSTD_CStream>::Create(
+		input, parameters, _stream);
+#else
+	return B_NOT_SUPPORTED;
+#endif
+}
+
+
+status_t
+BZstdCompressionAlgorithm::CreateCompressingOutputStream(BDataIO* output,
+	const BCompressionParameters* parameters, BDataIO*& _stream)
+{
+#ifdef B_ZSTD_COMPRESSION_SUPPORT
+	return Stream<BAbstractOutputStream, CompressionStrategy, ZSTD_CStream>::Create(
+		output, parameters, _stream);
+#else
+	return B_NOT_SUPPORTED;
+#endif
+}
+
+
+status_t
+BZstdCompressionAlgorithm::CreateDecompressingInputStream(BDataIO* input,
+	const BDecompressionParameters* parameters, BDataIO*& _stream)
+{
+#ifdef ZSTD_ENABLED
+	return Stream<BAbstractInputStream, DecompressionStrategy, ZSTD_DStream>::Create(
+		input, parameters, _stream);
+#else
+	return B_NOT_SUPPORTED;
+#endif
+}
+
+
+status_t
+BZstdCompressionAlgorithm::CreateDecompressingOutputStream(BDataIO* output,
+	const BDecompressionParameters* parameters, BDataIO*& _stream)
+{
+#ifdef ZSTD_ENABLED
+	return Stream<BAbstractOutputStream, DecompressionStrategy, ZSTD_DStream>::Create(
+		output, parameters, _stream);
+#else
+	return B_NOT_SUPPORTED;
+#endif
+}
+
+
+status_t
+BZstdCompressionAlgorithm::CompressBuffer(const void* input,
+	size_t inputSize, void* output, size_t outputSize, size_t& _compressedSize,
+	const BCompressionParameters* parameters)
+{
+#ifdef B_ZSTD_COMPRESSION_SUPPORT
+	const BZstdCompressionParameters* zstdParameters
+		= dynamic_cast<const BZstdCompressionParameters*>(parameters);
+	int compressionLevel = zstdParameters != NULL
+		? zstdParameters->CompressionLevel()
+		: B_ZSTD_COMPRESSION_DEFAULT;
+
+	size_t zstdError = ZSTD_compress(output, outputSize, input,
+		inputSize, compressionLevel);
+	if (ZSTD_isError(zstdError))
+		return _TranslateZstdError(zstdError);
+
+	_compressedSize = zstdError;
+	return B_OK;
+#else
+	return B_NOT_SUPPORTED;
+#endif
+}
+
+
+status_t
+BZstdCompressionAlgorithm::DecompressBuffer(const void* input,
+	size_t inputSize, void* output, size_t outputSize,
+	size_t& _uncompressedSize, const BDecompressionParameters* parameters)
+{
+#ifdef ZSTD_ENABLED
+	size_t zstdError = ZSTD_decompress(output, outputSize, input,
+		inputSize);
+	if (ZSTD_isError(zstdError))
+		return _TranslateZstdError(zstdError);
+
+	_uncompressedSize = zstdError;
+	return B_OK;
+#else
+	return B_NOT_SUPPORTED;
+#endif
+}
+
+
+/*static*/ status_t
+BZstdCompressionAlgorithm::_TranslateZstdError(size_t error)
+{
+#ifdef ZSTD_ENABLED
+	switch (ZSTD_getErrorCode(error)) {
+		case ZSTD_error_no_error:
+			return B_OK;
+		case ZSTD_error_seekableIO:
+			return B_BAD_VALUE;
+		case ZSTD_error_corruption_detected:
+		case ZSTD_error_checksum_wrong:
+			return B_BAD_DATA;
+		case ZSTD_error_version_unsupported:
+			return B_BAD_VALUE;
+		default:
+			return B_ERROR;
+	}
+#else
+	return B_NOT_SUPPORTED;
+#endif
+}

src/system/kernel/lib/Jamfile

@@ -141,3 +141,6 @@ KernelMergeObject kernel_misc.o :
 
 HaikuSubInclude arch $(TARGET_ARCH) ;
 HaikuSubInclude zlib ;
+if [ FIsBuildFeatureEnabled zstd ] {
+	HaikuSubInclude zstd ;
+}

src/system/kernel/lib/zstd/Jamfile

@@ -0,0 +1,26 @@
+SubDir HAIKU_TOP src system kernel lib zstd ;
+
+local zstdSourceDirectory = [ BuildFeatureAttribute zstd : sources : path ] ;
+UseHeaders [ FDirName $(zstdSourceDirectory) lib ] ;
+UseHeaders [ FDirName $(zstdSourceDirectory) lib common ] ;
+
+local zstdCommonSources =
+	error_private.c
+	entropy_common.c fse_decompress.c zstd_common.c
+	xxhash.c
+	;
+local zstdDecSources =
+	huf_decompress.c zstd_decompress.c
+	;
+
+LOCATE on [ FGristFiles $(zstdCommonSources) ] =
+	[ FDirName $(zstdSourceDirectory) lib common ] ;
+LOCATE on [ FGristFiles $(zstdDecSources) ] =
+	[ FDirName $(zstdSourceDirectory) lib decompress ] ;
+Depends [ FGristFiles $(zstdCommonSources) $(zstdDecSources) ]
+	: [ BuildFeatureAttribute zstd : sources ] ;
+
+# Build zstd with PIC, such that it can be used by kernel add-ons (filesystems).
+KernelStaticLibrary kernel_libzstd.a :
+	$(zstdCommonSources) $(zstdDecSources)
+	;

src/tests/kits/support/compression_test.cpp

@@ -13,6 +13,7 @@
 #include <File.h>
 
 #include <ZlibCompressionAlgorithm.h>
+#include <ZstdCompressionAlgorithm.h>
 
 
 extern const char* __progname;
@@ -22,6 +23,7 @@ const char* kCommandName = __progname;
 enum CompressionType {
 	ZlibCompression,
 	GzipCompression,
+	ZstdCompression,
 };
 
 
@@ -36,7 +38,8 @@ static const char* kUsage =
 	"  -d, --decompress\n"
 	"      Decompress the input file (default is compress).\n"
 	"  -f <format>\n"
-	"      Specify the compression format: \"zlib\" (default), or \"gzip\"\n"
+	"      Specify the compression format: \"zlib\" (default), \"gzip\"\n"
+	"      or \"zstd\".\n"
 	"  -h, --help\n"
 	"      Print this usage info.\n"
 	"  -i, --input-stream\n"
@@ -55,9 +58,9 @@ print_usage_and_exit(bool error)
 int
 main(int argc, const char* const* argv)
 {
-	int compressionLevel = B_ZLIB_COMPRESSION_DEFAULT;
+	int compressionLevel = -1;
 	bool compress = true;
-	bool useInputStream = true;
+	bool useInputStream = false;
 	CompressionType compressionType = ZlibCompression;
 
 	while (true) {
@@ -101,6 +104,8 @@ main(int argc, const char* const* argv)
 					compressionType = ZlibCompression;
 				} else if (strcmp(optarg, "gzip") == 0) {
 					compressionType = GzipCompression;
+				} else if (strcmp(optarg, "zstd") == 0) {
+					compressionType = ZstdCompression;
 				} else {
 					fprintf(stderr, "Error: Unsupported compression type "
 						"\"%s\"\n", optarg);
@@ -152,6 +157,8 @@ main(int argc, const char* const* argv)
 		case ZlibCompression:
 		case GzipCompression:
 		{
+			if (compressionLevel < 0)
+				compressionLevel = B_ZLIB_COMPRESSION_DEFAULT;
 			compressionAlgorithm = new BZlibCompressionAlgorithm;
 			BZlibCompressionParameters* zlibCompressionParameters
 				= new BZlibCompressionParameters(compressionLevel);
@@ -161,6 +168,16 @@ main(int argc, const char* const* argv)
 			decompressionParameters = new BZlibDecompressionParameters;
 			break;
 		}
+		case ZstdCompression:
+		{
+			if (compressionLevel < 0)
+				compressionLevel = B_ZSTD_COMPRESSION_DEFAULT;
+			compressionAlgorithm = new BZstdCompressionAlgorithm;
+			compressionParameters
+				= new BZstdCompressionParameters(compressionLevel);
+			decompressionParameters = new BZstdDecompressionParameters;
+			break;
+		}
 	}
 
 	if (useInputStream) {