Commit d1b3409df1804ff3182596ca41e2e2424c0918e7 | zig

author	Andrew Kelley <andrew@ziglang.org> 2022-11-15 20:21:50-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2022-12-06 12:15:04-07:00
log	d1b3409df1804ff3182596ca41e2e2424c0918e7
tree	906214ea90105f87d10695750f9c89af08121fcd
parent	39fd77bc163f0d7fc7408cbb4e48f960b4739448

add zstd v1.5.2

only the decompression files

34 files changed, 21147 insertions(+), 0 deletions(-)

stage1/zstd/LICENSE created+30

...	...	@@ -0,0 +1,30 @@
	1	BSD License
	2
	3	For Zstandard software
	4
	5	Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
	6
	7	Redistribution and use in source and binary forms, with or without modification,
	8	are permitted provided that the following conditions are met:
	9
	10	* Redistributions of source code must retain the above copyright notice, this
	11	list of conditions and the following disclaimer.
	12
	13	* Redistributions in binary form must reproduce the above copyright notice,
	14	this list of conditions and the following disclaimer in the documentation
	15	and/or other materials provided with the distribution.
	16
	17	* Neither the name Facebook nor the names of its contributors may be used to
	18	endorse or promote products derived from this software without specific
	19	prior written permission.
	20
	21	THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
	22	ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
	23	WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
	24	DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
	25	ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
	26	(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	27	LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
	28	ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	29	(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
	30	SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

stage1/zstd/lib/common/bitstream.h created+478

...	...	@@ -0,0 +1,478 @@
	1	/* ******************************************************************
	2	* bitstream
	3	* Part of FSE library
	4	* Copyright (c) Yann Collet, Facebook, Inc.
	5	*
	6	* You can contact the author at :
	7	* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
	8	*
	9	* This source code is licensed under both the BSD-style license (found in the
	10	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	11	* in the COPYING file in the root directory of this source tree).
	12	* You may select, at your option, one of the above-listed licenses.
	13	****************************************************************** */
	14	#ifndef BITSTREAM_H_MODULE
	15	#define BITSTREAM_H_MODULE
	16
	17	#if defined (__cplusplus)
	18	extern "C" {
	19	#endif
	20	/*
	21	* This API consists of small unitary functions, which must be inlined for best performance.
	22	* Since link-time-optimization is not available for all compilers,
	23	* these functions are defined into a .h to be included.
	24	*/
	25
	26	/-***************************************
	27	* Dependencies
	28	******************************************/
	29	#include "mem.h" /* unaligned access routines */
	30	#include "compiler.h" /* UNLIKELY() */
	31	#include "debug.h" /* assert(), DEBUGLOG(), RAWLOG() */
	32	#include "error_private.h" /* error codes and messages */
	33
	34
	35	/*=========================================
	36	* Target specific
	37	=========================================*/
	38	#ifndef ZSTD_NO_INTRINSICS
	39	# if defined(__BMI__) && defined(__GNUC__)
	40	# include <immintrin.h> /* support for bextr (experimental) */
	41	# elif defined(__ICCARM__)
	42	# include <intrinsics.h>
	43	# endif
	44	#endif
	45
	46	#define STREAM_ACCUMULATOR_MIN_32 25
	47	#define STREAM_ACCUMULATOR_MIN_64 57
	48	#define STREAM_ACCUMULATOR_MIN ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64))
	49
	50
	51	/-*****************************************
	52	* bitStream encoding API (write forward)
	53	********************************************/
	54	/* bitStream can mix input from multiple sources.
	55	* A critical property of these streams is that they encode and decode in reverse direction.
	56	* So the first bit sequence you add will be the last to be read, like a LIFO stack.
	57	*/
	58	typedef struct {
	59	size_t bitContainer;
	60	unsigned bitPos;
	61	char* startPtr;
	62	char* ptr;
	63	char* endPtr;
	64	} BIT_CStream_t;
	65
	66	MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity);
	67	MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
	68	MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC);
	69	MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
	70
	71	/* Start with initCStream, providing the size of buffer to write into.
	72	* bitStream will never write outside of this buffer.
	73	* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
	74	*
	75	* bits are first added to a local register.
	76	* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
	77	* Writing data into memory is an explicit operation, performed by the flushBits function.
	78	* Hence keep track how many bits are potentially stored into local register to avoid register overflow.
	79	* After a flushBits, a maximum of 7 bits might still be stored into local register.
	80	*
	81	* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers.
	82	*
	83	* Last operation is to close the bitStream.
	84	* The function returns the final size of CStream in bytes.
	85	* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable)
	86	*/
	87
	88
	89	/-*******************************************
	90	* bitStream decoding API (read backward)
	91	**********************************************/
	92	typedef struct {
	93	size_t bitContainer;
	94	unsigned bitsConsumed;
	95	const char* ptr;
	96	const char* start;
	97	const char* limitPtr;
	98	} BIT_DStream_t;
	99
	100	typedef enum { BIT_DStream_unfinished = 0,
	101	BIT_DStream_endOfBuffer = 1,
	102	BIT_DStream_completed = 2,
	103	BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */
	104	/* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
	105
	106	MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
	107	MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
	108	MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
	109	MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
	110
	111
	112	/* Start by invoking BIT_initDStream().
	113	* A chunk of the bitStream is then stored into a local register.
	114	* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
	115	* You can then retrieve bitFields stored into the local register, in reverse order.
	116	* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
	117	* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
	118	* Otherwise, it can be less than that, so proceed accordingly.
	119	* Checking if DStream has reached its end can be performed with BIT_endOfDStream().
	120	*/
	121
	122
	123	/-***************************************
	124	* unsafe API
	125	******************************************/
	126	MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
	127	/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
	128
	129	MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC);
	130	/* unsafe version; does not check buffer overflow */
	131
	132	MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
	133	/* faster, but works only if nbBits >= 1 */
	134
	135
	136
	137	/-*************************************************************
	138	* Internal functions
	139	****************************************************************/
	140	MEM_STATIC unsigned BIT_highbit32 (U32 val)
	141	{
	142	assert(val != 0);
	143	{
	144	# if defined(_MSC_VER) /* Visual */
	145	# if STATIC_BMI2 == 1
	146	return _lzcnt_u32(val) ^ 31;
	147	# else
	148	if (val != 0) {
	149	unsigned long r;
	150	_BitScanReverse(&r, val);
	151	return (unsigned)r;
	152	} else {
	153	/* Should not reach this code path */
	154	__assume(0);
	155	}
	156	# endif
	157	# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
	158	return __builtin_clz (val) ^ 31;
	159	# elif defined(__ICCARM__) /* IAR Intrinsic */
	160	return 31 - __CLZ(val);
	161	# else /* Software version */
	162	static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29,
	163	11, 14, 16, 18, 22, 25, 3, 30,
	164	8, 12, 20, 28, 15, 17, 24, 7,
	165	19, 27, 23, 6, 26, 5, 4, 31 };
	166	U32 v = val;
	167	v \|= v >> 1;
	168	v \|= v >> 2;
	169	v \|= v >> 4;
	170	v \|= v >> 8;
	171	v \|= v >> 16;
	172	return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
	173	# endif
	174	}
	175	}
	176
	177	/===== Local Constants =====/
	178	static const unsigned BIT_mask[] = {
	179	0, 1, 3, 7, 0xF, 0x1F,
	180	0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF,
	181	0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF,
	182	0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF,
	183	0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF,
	184	0x3FFFFFFF, 0x7FFFFFFF}; /* up to 31 bits */
	185	#define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0]))
	186
	187	/-*************************************************************
	188	* bitStream encoding
	189	****************************************************************/
	190	/*! BIT_initCStream() :
	191	* `dstCapacity` must be > sizeof(size_t)
	192	* @return : 0 if success,
	193	* otherwise an error code (can be tested using ERR_isError()) */
	194	MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
	195	void* startPtr, size_t dstCapacity)
	196	{
	197	bitC->bitContainer = 0;
	198	bitC->bitPos = 0;
	199	bitC->startPtr = (char*)startPtr;
	200	bitC->ptr = bitC->startPtr;
	201	bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer);
	202	if (dstCapacity <= sizeof(bitC->bitContainer)) return ERROR(dstSize_tooSmall);
	203	return 0;
	204	}
	205
	206	/*! BIT_addBits() :
	207	* can add up to 31 bits into `bitC`.
	208	* Note : does not check for register overflow ! */
	209	MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
	210	size_t value, unsigned nbBits)
	211	{
	212	DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32);
	213	assert(nbBits < BIT_MASK_SIZE);
	214	assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
	215	bitC->bitContainer \|= (value & BIT_mask[nbBits]) << bitC->bitPos;
	216	bitC->bitPos += nbBits;
	217	}
	218
	219	/*! BIT_addBitsFast() :
	220	* works only if `value` is _clean_,
	221	* meaning all high bits above nbBits are 0 */
	222	MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,
	223	size_t value, unsigned nbBits)
	224	{
	225	assert((value>>nbBits) == 0);
	226	assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
	227	bitC->bitContainer \|= value << bitC->bitPos;
	228	bitC->bitPos += nbBits;
	229	}
	230
	231	/*! BIT_flushBitsFast() :
	232	* assumption : bitContainer has not overflowed
	233	* unsafe version; does not check buffer overflow */
	234	MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
	235	{
	236	size_t const nbBytes = bitC->bitPos >> 3;
	237	assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);
	238	assert(bitC->ptr <= bitC->endPtr);
	239	MEM_writeLEST(bitC->ptr, bitC->bitContainer);
	240	bitC->ptr += nbBytes;
	241	bitC->bitPos &= 7;
	242	bitC->bitContainer >>= nbBytes*8;
	243	}
	244
	245	/*! BIT_flushBits() :
	246	* assumption : bitContainer has not overflowed
	247	* safe version; check for buffer overflow, and prevents it.
	248	* note : does not signal buffer overflow.
	249	* overflow will be revealed later on using BIT_closeCStream() */
	250	MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC)
	251	{
	252	size_t const nbBytes = bitC->bitPos >> 3;
	253	assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);
	254	assert(bitC->ptr <= bitC->endPtr);
	255	MEM_writeLEST(bitC->ptr, bitC->bitContainer);
	256	bitC->ptr += nbBytes;
	257	if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
	258	bitC->bitPos &= 7;
	259	bitC->bitContainer >>= nbBytes*8;
	260	}
	261
	262	/*! BIT_closeCStream() :
	263	* @return : size of CStream, in bytes,
	264	* or 0 if it could not fit into dstBuffer */
	265	MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
	266	{
	267	BIT_addBitsFast(bitC, 1, 1); /* endMark */
	268	BIT_flushBits(bitC);
	269	if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */
	270	return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
	271	}
	272
	273
	274	/-*******************************************************
	275	* bitStream decoding
	276	**********************************************************/
	277	/*! BIT_initDStream() :
	278	* Initialize a BIT_DStream_t.
	279	* `bitD` : a pointer to an already allocated BIT_DStream_t structure.
	280	* `srcSize` must be the exact size of the bitStream, in bytes.
	281	* @return : size of stream (== srcSize), or an errorCode if a problem is detected
	282	*/
	283	MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
	284	{
	285	if (srcSize < 1) { ZSTD_memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }
	286
	287	bitD->start = (const char*)srcBuffer;
	288	bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer);
	289
	290	if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */
	291	bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
	292	bitD->bitContainer = MEM_readLEST(bitD->ptr);
	293	{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
	294	bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */
	295	if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
	296	} else {
	297	bitD->ptr = bitD->start;
	298	bitD->bitContainer = (const BYTE)(bitD->start);
	299	switch(srcSize)
	300	{
	301	case 7: bitD->bitContainer += (size_t)(((const BYTE)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)8 - 16);
	302	ZSTD_FALLTHROUGH;
	303
	304	case 6: bitD->bitContainer += (size_t)(((const BYTE)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)8 - 24);
	305	ZSTD_FALLTHROUGH;
	306
	307	case 5: bitD->bitContainer += (size_t)(((const BYTE)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)8 - 32);
	308	ZSTD_FALLTHROUGH;
	309
	310	case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
	311	ZSTD_FALLTHROUGH;
	312
	313	case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
	314	ZSTD_FALLTHROUGH;
	315
	316	case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8;
	317	ZSTD_FALLTHROUGH;
	318
	319	default: break;
	320	}
	321	{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
	322	bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
	323	if (lastByte == 0) return ERROR(corruption_detected); /* endMark not present */
	324	}
	325	bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8;
	326	}
	327
	328	return srcSize;
	329	}
	330
	331	MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
	332	{
	333	return bitContainer >> start;
	334	}
	335
	336	MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
	337	{
	338	U32 const regMask = sizeof(bitContainer)*8 - 1;
	339	/* if start > regMask, bitstream is corrupted, and result is undefined */
	340	assert(nbBits < BIT_MASK_SIZE);
	341	/* x86 transform & ((1 << nbBits) - 1) to bzhi instruction, it is better
	342	* than accessing memory. When bmi2 instruction is not present, we consider
	343	* such cpus old (pre-Haswell, 2013) and their performance is not of that
	344	* importance.
	345	*/
	346	#if defined(__x86_64__) \|\| defined(_M_X86)
	347	return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1);
	348	#else
	349	return (bitContainer >> (start & regMask)) & BIT_mask[nbBits];
	350	#endif
	351	}
	352
	353	MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
	354	{
	355	#if defined(STATIC_BMI2) && STATIC_BMI2 == 1
	356	return _bzhi_u64(bitContainer, nbBits);
	357	#else
	358	assert(nbBits < BIT_MASK_SIZE);
	359	return bitContainer & BIT_mask[nbBits];
	360	#endif
	361	}
	362
	363	/*! BIT_lookBits() :
	364	* Provides next n bits from local register.
	365	* local register is not modified.
	366	* On 32-bits, maxNbBits==24.
	367	* On 64-bits, maxNbBits==56.
	368	* @return : value extracted */
	369	MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
	370	{
	371	/* arbitrate between double-shift and shift+mask */
	372	#if 1
	373	/* if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8,
	374	* bitstream is likely corrupted, and result is undefined */
	375	return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
	376	#else
	377	/* this code path is slower on my os-x laptop */
	378	U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
	379	return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask);
	380	#endif
	381	}
	382
	383	/*! BIT_lookBitsFast() :
	384	* unsafe version; only works if nbBits >= 1 */
	385	MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
	386	{
	387	U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
	388	assert(nbBits >= 1);
	389	return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask);
	390	}
	391
	392	MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
	393	{
	394	bitD->bitsConsumed += nbBits;
	395	}
	396
	397	/*! BIT_readBits() :
	398	* Read (consume) next n bits from local register and update.
	399	* Pay attention to not read more than nbBits contained into local register.
	400	* @return : extracted value. */
	401	MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
	402	{
	403	size_t const value = BIT_lookBits(bitD, nbBits);
	404	BIT_skipBits(bitD, nbBits);
	405	return value;
	406	}
	407
	408	/*! BIT_readBitsFast() :
	409	* unsafe version; only works only if nbBits >= 1 */
	410	MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
	411	{
	412	size_t const value = BIT_lookBitsFast(bitD, nbBits);
	413	assert(nbBits >= 1);
	414	BIT_skipBits(bitD, nbBits);
	415	return value;
	416	}
	417
	418	/*! BIT_reloadDStreamFast() :
	419	* Similar to BIT_reloadDStream(), but with two differences:
	420	* 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold!
	421	* 2. Returns BIT_DStream_overflow when bitD->ptr < bitD->limitPtr, at this
	422	* point you must use BIT_reloadDStream() to reload.
	423	*/
	424	MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD)
	425	{
	426	if (UNLIKELY(bitD->ptr < bitD->limitPtr))
	427	return BIT_DStream_overflow;
	428	assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8);
	429	bitD->ptr -= bitD->bitsConsumed >> 3;
	430	bitD->bitsConsumed &= 7;
	431	bitD->bitContainer = MEM_readLEST(bitD->ptr);
	432	return BIT_DStream_unfinished;
	433	}
	434
	435	/*! BIT_reloadDStream() :
	436	* Refill `bitD` from buffer previously set in BIT_initDStream() .
	437	* This function is safe, it guarantees it will not read beyond src buffer.
	438	* @return : status of `BIT_DStream_t` internal register.
	439	* when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */
	440	MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
	441	{
	442	if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)8)) / overflow detected, like end of stream */
	443	return BIT_DStream_overflow;
	444
	445	if (bitD->ptr >= bitD->limitPtr) {
	446	return BIT_reloadDStreamFast(bitD);
	447	}
	448	if (bitD->ptr == bitD->start) {
	449	if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer;
	450	return BIT_DStream_completed;
	451	}
	452	/* start < ptr < limitPtr */
	453	{ U32 nbBytes = bitD->bitsConsumed >> 3;
	454	BIT_DStream_status result = BIT_DStream_unfinished;
	455	if (bitD->ptr - nbBytes < bitD->start) {
	456	nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */
	457	result = BIT_DStream_endOfBuffer;
	458	}
	459	bitD->ptr -= nbBytes;
	460	bitD->bitsConsumed -= nbBytes*8;
	461	bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD->bitContainer), otherwise bitD->ptr == bitD->start */
	462	return result;
	463	}
	464	}
	465
	466	/*! BIT_endOfDStream() :
	467	* @return : 1 if DStream has _exactly_ reached its end (all bits consumed).
	468	*/
	469	MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
	470	{
	471	return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));
	472	}
	473
	474	#if defined (__cplusplus)
	475	}
	476	#endif
	477
	478	#endif /* BITSTREAM_H_MODULE */

stage1/zstd/lib/common/compiler.h created+335

...	...	@@ -0,0 +1,335 @@
	1	/*
	2	* Copyright (c) Yann Collet, Facebook, Inc.
	3	* All rights reserved.
	4	*
	5	* This source code is licensed under both the BSD-style license (found in the
	6	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	7	* in the COPYING file in the root directory of this source tree).
	8	* You may select, at your option, one of the above-listed licenses.
	9	*/
	10
	11	#ifndef ZSTD_COMPILER_H
	12	#define ZSTD_COMPILER_H
	13
	14	#include "portability_macros.h"
	15
	16	/-******************************************************
	17	* Compiler specifics
	18	*********************************************************/
	19	/* force inlining */
	20
	21	#if !defined(ZSTD_NO_INLINE)
	22	#if (defined(__GNUC__) && !defined(__STRICT_ANSI__)) \|\| defined(__cplusplus) \|\| defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
	23	# define INLINE_KEYWORD inline
	24	#else
	25	# define INLINE_KEYWORD
	26	#endif
	27
	28	#if defined(__GNUC__) \|\| defined(__ICCARM__)
	29	# define FORCE_INLINE_ATTR __attribute__((always_inline))
	30	#elif defined(_MSC_VER)
	31	# define FORCE_INLINE_ATTR __forceinline
	32	#else
	33	# define FORCE_INLINE_ATTR
	34	#endif
	35
	36	#else
	37
	38	#define INLINE_KEYWORD
	39	#define FORCE_INLINE_ATTR
	40
	41	#endif
	42
	43	/**
	44	On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC).
	45	This explicitly marks such functions as __cdecl so that the code will still compile
	46	if a CC other than __cdecl has been made the default.
	47	*/
	48	#if defined(_MSC_VER)
	49	# define WIN_CDECL __cdecl
	50	#else
	51	# define WIN_CDECL
	52	#endif
	53
	54	/**
	55	* FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant
	56	* parameters. They must be inlined for the compiler to eliminate the constant
	57	* branches.
	58	*/
	59	#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR
	60	/**
	61	* HINT_INLINE is used to help the compiler generate better code. It is not
	62	* used for "templates", so it can be tweaked based on the compilers
	63	* performance.
	64	*
	65	* gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the
	66	* always_inline attribute.
	67	*
	68	* clang up to 5.0.0 (trunk) benefit tremendously from the always_inline
	69	* attribute.
	70	*/
	71	#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5
	72	# define HINT_INLINE static INLINE_KEYWORD
	73	#else
	74	# define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR
	75	#endif
	76
	77	/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */
	78	#if defined(__GNUC__)
	79	# define UNUSED_ATTR __attribute__((unused))
	80	#else
	81	# define UNUSED_ATTR
	82	#endif
	83
	84	/* force no inlining */
	85	#ifdef _MSC_VER
	86	# define FORCE_NOINLINE static __declspec(noinline)
	87	#else
	88	# if defined(__GNUC__) \|\| defined(__ICCARM__)
	89	# define FORCE_NOINLINE static __attribute__((__noinline__))
	90	# else
	91	# define FORCE_NOINLINE static
	92	# endif
	93	#endif
	94
	95
	96	/* target attribute */
	97	#if defined(__GNUC__) \|\| defined(__ICCARM__)
	98	# define TARGET_ATTRIBUTE(target) __attribute__((__target__(target)))
	99	#else
	100	# define TARGET_ATTRIBUTE(target)
	101	#endif
	102
	103	/* Target attribute for BMI2 dynamic dispatch.
	104	* Enable lzcnt, bmi, and bmi2.
	105	* We test for bmi1 & bmi2. lzcnt is included in bmi1.
	106	*/
	107	#define BMI2_TARGET_ATTRIBUTE TARGET_ATTRIBUTE("lzcnt,bmi,bmi2")
	108
	109	/* prefetch
	110	* can be disabled, by declaring NO_PREFETCH build macro */
	111	#if defined(NO_PREFETCH)
	112	# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */
	113	# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */
	114	#else
	115	# if defined(_MSC_VER) && (defined(_M_X64) \|\| defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */
	116	# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
	117	# define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
	118	# define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
	119	# elif defined(__GNUC__) && ( (__GNUC__ >= 4) \|\| ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
	120	# define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read /, 3 / locality */)
	121	# define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read /, 2 / locality */)
	122	# elif defined(__aarch64__)
	123	# define PREFETCH_L1(ptr) __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr)))
	124	# define PREFETCH_L2(ptr) __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr)))
	125	# else
	126	# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */
	127	# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */
	128	# endif
	129	#endif /* NO_PREFETCH */
	130
	131	#define CACHELINE_SIZE 64
	132
	133	#define PREFETCH_AREA(p, s) { \
	134	const char* const _ptr = (const char*)(p); \
	135	size_t const _size = (size_t)(s); \
	136	size_t _pos; \
	137	for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \
	138	PREFETCH_L2(_ptr + _pos); \
	139	} \
	140	}
	141
	142	/* vectorization
	143	* older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax,
	144	* and some compilers, like Intel ICC and MCST LCC, do not support it at all. */
	145	#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) && !defined(__LCC__)
	146	# if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) \|\| (__GNUC__ >= 5)
	147	# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize")))
	148	# else
	149	# define DONT_VECTORIZE _Pragma("GCC optimize(\"no-tree-vectorize\")")
	150	# endif
	151	#else
	152	# define DONT_VECTORIZE
	153	#endif
	154
	155	/* Tell the compiler that a branch is likely or unlikely.
	156	* Only use these macros if it causes the compiler to generate better code.
	157	* If you can remove a LIKELY/UNLIKELY annotation without speed changes in gcc
	158	* and clang, please do.
	159	*/
	160	#if defined(__GNUC__)
	161	#define LIKELY(x) (__builtin_expect((x), 1))
	162	#define UNLIKELY(x) (__builtin_expect((x), 0))
	163	#else
	164	#define LIKELY(x) (x)
	165	#define UNLIKELY(x) (x)
	166	#endif
	167
	168	/* disable warnings */
	169	#ifdef _MSC_VER /* Visual Studio */
	170	# include <intrin.h> /* For Visual 2005 */
	171	# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */
	172	# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
	173	# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
	174	# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
	175	# pragma warning(disable : 4324) /* disable: C4324: padded structure */
	176	#endif
	177
	178	/Like DYNAMIC_BMI2 but for compile time determination of BMI2 support/
	179	#ifndef STATIC_BMI2
	180	# if defined(_MSC_VER) && (defined(_M_X64) \|\| defined(_M_I86))
	181	# ifdef __AVX2__ //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2
	182	# define STATIC_BMI2 1
	183	# endif
	184	# endif
	185	#endif
	186
	187	#ifndef STATIC_BMI2
	188	#define STATIC_BMI2 0
	189	#endif
	190
	191	/* compile time determination of SIMD support */
	192	#if !defined(ZSTD_NO_INTRINSICS)
	193	# if defined(__SSE2__) \|\| defined(_M_AMD64) \|\| (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2))
	194	# define ZSTD_ARCH_X86_SSE2
	195	# endif
	196	# if defined(__ARM_NEON) \|\| defined(_M_ARM64)
	197	# define ZSTD_ARCH_ARM_NEON
	198	# endif
	199	#
	200	# if defined(ZSTD_ARCH_X86_SSE2)
	201	# include <emmintrin.h>
	202	# elif defined(ZSTD_ARCH_ARM_NEON)
	203	# include <arm_neon.h>
	204	# endif
	205	#endif
	206
	207	/* C-language Attributes are added in C23. */
	208	#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute)
	209	# define ZSTD_HAS_C_ATTRIBUTE(x) __has_c_attribute(x)
	210	#else
	211	# define ZSTD_HAS_C_ATTRIBUTE(x) 0
	212	#endif
	213
	214	/* Only use C++ attributes in C++. Some compilers report support for C++
	215	* attributes when compiling with C.
	216	*/
	217	#if defined(__cplusplus) && defined(__has_cpp_attribute)
	218	# define ZSTD_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
	219	#else
	220	# define ZSTD_HAS_CPP_ATTRIBUTE(x) 0
	221	#endif
	222
	223	/* Define ZSTD_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute.
	224	* - C23: https://en.cppreference.com/w/c/language/attributes/fallthrough
	225	* - CPP17: https://en.cppreference.com/w/cpp/language/attributes/fallthrough
	226	* - Else: __attribute__((__fallthrough__))
	227	*/
	228	#ifndef ZSTD_FALLTHROUGH
	229	# if ZSTD_HAS_C_ATTRIBUTE(fallthrough)
	230	# define ZSTD_FALLTHROUGH [[fallthrough]]
	231	# elif ZSTD_HAS_CPP_ATTRIBUTE(fallthrough)
	232	# define ZSTD_FALLTHROUGH [[fallthrough]]
	233	# elif __has_attribute(__fallthrough__)
	234	/* Leading semicolon is to satisfy gcc-11 with -pedantic. Without the semicolon
	235	* gcc complains about: a label can only be part of a statement and a declaration is not a statement.
	236	*/
	237	# define ZSTD_FALLTHROUGH ; __attribute__((__fallthrough__))
	238	# else
	239	# define ZSTD_FALLTHROUGH
	240	# endif
	241	#endif
	242
	243	/-*************************************************************
	244	* Alignment check
	245	*****************************************************************/
	246
	247	/* this test was initially positioned in mem.h,
	248	* but this file is removed (or replaced) for linux kernel
	249	* so it's now hosted in compiler.h,
	250	* which remains valid for both user & kernel spaces.
	251	*/
	252
	253	#ifndef ZSTD_ALIGNOF
	254	# if defined(__GNUC__) \|\| defined(_MSC_VER)
	255	/* covers gcc, clang & MSVC */
	256	/* note : this section must come first, before C11,
	257	* due to a limitation in the kernel source generator */
	258	# define ZSTD_ALIGNOF(T) __alignof(T)
	259
	260	# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
	261	/* C11 support */
	262	# include <stdalign.h>
	263	# define ZSTD_ALIGNOF(T) alignof(T)
	264
	265	# else
	266	/* No known support for alignof() - imperfect backup */
	267	# define ZSTD_ALIGNOF(T) (sizeof(void) < sizeof(T) ? sizeof(void) : sizeof(T))
	268
	269	# endif
	270	#endif /* ZSTD_ALIGNOF */
	271
	272	/-*************************************************************
	273	* Sanitizer
	274	*****************************************************************/
	275
	276	#if ZSTD_MEMORY_SANITIZER
	277	/* Not all platforms that support msan provide sanitizers/msan_interface.h.
	278	* We therefore declare the functions we need ourselves, rather than trying to
	279	* include the header file... */
	280	#include <stddef.h> /* size_t */
	281	#define ZSTD_DEPS_NEED_STDINT
	282	#include "zstd_deps.h" /* intptr_t */
	283
	284	/* Make memory region fully initialized (without changing its contents). */
	285	void __msan_unpoison(const volatile void *a, size_t size);
	286
	287	/* Make memory region fully uninitialized (without changing its contents).
	288	This is a legacy interface that does not update origin information. Use
	289	__msan_allocated_memory() instead. */
	290	void __msan_poison(const volatile void *a, size_t size);
	291
	292	/* Returns the offset of the first (at least partially) poisoned byte in the
	293	memory range, or -1 if the whole range is good. */
	294	intptr_t __msan_test_shadow(const volatile void *x, size_t size);
	295	#endif
	296
	297	#if ZSTD_ADDRESS_SANITIZER
	298	/* Not all platforms that support asan provide sanitizers/asan_interface.h.
	299	* We therefore declare the functions we need ourselves, rather than trying to
	300	* include the header file... */
	301	#include <stddef.h> /* size_t */
	302
	303	/**
	304	* Marks a memory region (<c>[addr, addr+size)</c>) as unaddressable.
	305	*
	306	* This memory must be previously allocated by your program. Instrumented
	307	* code is forbidden from accessing addresses in this region until it is
	308	* unpoisoned. This function is not guaranteed to poison the entire region -
	309	* it could poison only a subregion of <c>[addr, addr+size)</c> due to ASan
	310	* alignment restrictions.
	311	*
	312	* \note This function is not thread-safe because no two threads can poison or
	313	* unpoison memory in the same memory region simultaneously.
	314	*
	315	* \param addr Start of memory region.
	316	* \param size Size of memory region. */
	317	void __asan_poison_memory_region(void const volatile *addr, size_t size);
	318
	319	/**
	320	* Marks a memory region (<c>[addr, addr+size)</c>) as addressable.
	321	*
	322	* This memory must be previously allocated by your program. Accessing
	323	* addresses in this region is allowed until this region is poisoned again.
	324	* This function could unpoison a super-region of <c>[addr, addr+size)</c> due
	325	* to ASan alignment restrictions.
	326	*
	327	* \note This function is not thread-safe because no two threads can
	328	* poison or unpoison memory in the same memory region simultaneously.
	329	*
	330	* \param addr Start of memory region.
	331	* \param size Size of memory region. */
	332	void __asan_unpoison_memory_region(void const volatile *addr, size_t size);
	333	#endif
	334
	335	#endif /* ZSTD_COMPILER_H */

stage1/zstd/lib/common/cpu.h created+213

...	...	@@ -0,0 +1,213 @@
	1	/*
	2	* Copyright (c) Facebook, Inc.
	3	* All rights reserved.
	4	*
	5	* This source code is licensed under both the BSD-style license (found in the
	6	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	7	* in the COPYING file in the root directory of this source tree).
	8	* You may select, at your option, one of the above-listed licenses.
	9	*/
	10
	11	#ifndef ZSTD_COMMON_CPU_H
	12	#define ZSTD_COMMON_CPU_H
	13
	14	/**
	15	* Implementation taken from folly/CpuId.h
	16	* https://github.com/facebook/folly/blob/master/folly/CpuId.h
	17	*/
	18
	19	#include "mem.h"
	20
	21	#ifdef _MSC_VER
	22	#include <intrin.h>
	23	#endif
	24
	25	typedef struct {
	26	U32 f1c;
	27	U32 f1d;
	28	U32 f7b;
	29	U32 f7c;
	30	} ZSTD_cpuid_t;
	31
	32	MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
	33	U32 f1c = 0;
	34	U32 f1d = 0;
	35	U32 f7b = 0;
	36	U32 f7c = 0;
	37	#if defined(_MSC_VER) && (defined(_M_X64) \|\| defined(_M_IX86))
	38	int reg[4];
	39	__cpuid((int*)reg, 0);
	40	{
	41	int const n = reg[0];
	42	if (n >= 1) {
	43	__cpuid((int*)reg, 1);
	44	f1c = (U32)reg[2];
	45	f1d = (U32)reg[3];
	46	}
	47	if (n >= 7) {
	48	__cpuidex((int*)reg, 7, 0);
	49	f7b = (U32)reg[1];
	50	f7c = (U32)reg[2];
	51	}
	52	}
	53	#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__)
	54	/* The following block like the normal cpuid branch below, but gcc
	55	* reserves ebx for use of its pic register so we must specially
	56	* handle the save and restore to avoid clobbering the register
	57	*/
	58	U32 n;
	59	__asm__(
	60	"pushl %%ebx\n\t"
	61	"cpuid\n\t"
	62	"popl %%ebx\n\t"
	63	: "=a"(n)
	64	: "a"(0)
	65	: "ecx", "edx");
	66	if (n >= 1) {
	67	U32 f1a;
	68	__asm__(
	69	"pushl %%ebx\n\t"
	70	"cpuid\n\t"
	71	"popl %%ebx\n\t"
	72	: "=a"(f1a), "=c"(f1c), "=d"(f1d)
	73	: "a"(1));
	74	}
	75	if (n >= 7) {
	76	__asm__(
	77	"pushl %%ebx\n\t"
	78	"cpuid\n\t"
	79	"movl %%ebx, %%eax\n\t"
	80	"popl %%ebx"
	81	: "=a"(f7b), "=c"(f7c)
	82	: "a"(7), "c"(0)
	83	: "edx");
	84	}
	85	#elif defined(__x86_64__) \|\| defined(_M_X64) \|\| defined(__i386__)
	86	U32 n;
	87	__asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx");
	88	if (n >= 1) {
	89	U32 f1a;
	90	__asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx");
	91	}
	92	if (n >= 7) {
	93	U32 f7a;
	94	__asm__("cpuid"
	95	: "=a"(f7a), "=b"(f7b), "=c"(f7c)
	96	: "a"(7), "c"(0)
	97	: "edx");
	98	}
	99	#endif
	100	{
	101	ZSTD_cpuid_t cpuid;
	102	cpuid.f1c = f1c;
	103	cpuid.f1d = f1d;
	104	cpuid.f7b = f7b;
	105	cpuid.f7c = f7c;
	106	return cpuid;
	107	}
	108	}
	109
	110	#define X(name, r, bit) \
	111	MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) { \
	112	return ((cpuid.r) & (1U << bit)) != 0; \
	113	}
	114
	115	/* cpuid(1): Processor Info and Feature Bits. */
	116	#define C(name, bit) X(name, f1c, bit)
	117	C(sse3, 0)
	118	C(pclmuldq, 1)
	119	C(dtes64, 2)
	120	C(monitor, 3)
	121	C(dscpl, 4)
	122	C(vmx, 5)
	123	C(smx, 6)
	124	C(eist, 7)
	125	C(tm2, 8)
	126	C(ssse3, 9)
	127	C(cnxtid, 10)
	128	C(fma, 12)
	129	C(cx16, 13)
	130	C(xtpr, 14)
	131	C(pdcm, 15)
	132	C(pcid, 17)
	133	C(dca, 18)
	134	C(sse41, 19)
	135	C(sse42, 20)
	136	C(x2apic, 21)
	137	C(movbe, 22)
	138	C(popcnt, 23)
	139	C(tscdeadline, 24)
	140	C(aes, 25)
	141	C(xsave, 26)
	142	C(osxsave, 27)
	143	C(avx, 28)
	144	C(f16c, 29)
	145	C(rdrand, 30)
	146	#undef C
	147	#define D(name, bit) X(name, f1d, bit)
	148	D(fpu, 0)
	149	D(vme, 1)
	150	D(de, 2)
	151	D(pse, 3)
	152	D(tsc, 4)
	153	D(msr, 5)
	154	D(pae, 6)
	155	D(mce, 7)
	156	D(cx8, 8)
	157	D(apic, 9)
	158	D(sep, 11)
	159	D(mtrr, 12)
	160	D(pge, 13)
	161	D(mca, 14)
	162	D(cmov, 15)
	163	D(pat, 16)
	164	D(pse36, 17)
	165	D(psn, 18)
	166	D(clfsh, 19)
	167	D(ds, 21)
	168	D(acpi, 22)
	169	D(mmx, 23)
	170	D(fxsr, 24)
	171	D(sse, 25)
	172	D(sse2, 26)
	173	D(ss, 27)
	174	D(htt, 28)
	175	D(tm, 29)
	176	D(pbe, 31)
	177	#undef D
	178
	179	/* cpuid(7): Extended Features. */
	180	#define B(name, bit) X(name, f7b, bit)
	181	B(bmi1, 3)
	182	B(hle, 4)
	183	B(avx2, 5)
	184	B(smep, 7)
	185	B(bmi2, 8)
	186	B(erms, 9)
	187	B(invpcid, 10)
	188	B(rtm, 11)
	189	B(mpx, 14)
	190	B(avx512f, 16)
	191	B(avx512dq, 17)
	192	B(rdseed, 18)
	193	B(adx, 19)
	194	B(smap, 20)
	195	B(avx512ifma, 21)
	196	B(pcommit, 22)
	197	B(clflushopt, 23)
	198	B(clwb, 24)
	199	B(avx512pf, 26)
	200	B(avx512er, 27)
	201	B(avx512cd, 28)
	202	B(sha, 29)
	203	B(avx512bw, 30)
	204	B(avx512vl, 31)
	205	#undef B
	206	#define C(name, bit) X(name, f7c, bit)
	207	C(prefetchwt1, 0)
	208	C(avx512vbmi, 1)
	209	#undef C
	210
	211	#undef X
	212
	213	#endif /* ZSTD_COMMON_CPU_H */

stage1/zstd/lib/common/debug.c created+24

...	...	@@ -0,0 +1,24 @@
	1	/* ******************************************************************
	2	* debug
	3	* Part of FSE library
	4	* Copyright (c) Yann Collet, Facebook, Inc.
	5	*
	6	* You can contact the author at :
	7	* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
	8	*
	9	* This source code is licensed under both the BSD-style license (found in the
	10	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	11	* in the COPYING file in the root directory of this source tree).
	12	* You may select, at your option, one of the above-listed licenses.
	13	****************************************************************** */
	14
	15
	16	/*
	17	* This module only hosts one global variable
	18	* which can be used to dynamically influence the verbosity of traces,
	19	* such as DEBUGLOG and RAWLOG
	20	*/
	21
	22	#include "debug.h"
	23
	24	int g_debuglevel = DEBUGLEVEL;

stage1/zstd/lib/common/debug.h created+107

...	...	@@ -0,0 +1,107 @@
	1	/* ******************************************************************
	2	* debug
	3	* Part of FSE library
	4	* Copyright (c) Yann Collet, Facebook, Inc.
	5	*
	6	* You can contact the author at :
	7	* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
	8	*
	9	* This source code is licensed under both the BSD-style license (found in the
	10	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	11	* in the COPYING file in the root directory of this source tree).
	12	* You may select, at your option, one of the above-listed licenses.
	13	****************************************************************** */
	14
	15
	16	/*
	17	* The purpose of this header is to enable debug functions.
	18	* They regroup assert(), DEBUGLOG() and RAWLOG() for run-time,
	19	* and DEBUG_STATIC_ASSERT() for compile-time.
	20	*
	21	* By default, DEBUGLEVEL==0, which means run-time debug is disabled.
	22	*
	23	* Level 1 enables assert() only.
	24	* Starting level 2, traces can be generated and pushed to stderr.
	25	* The higher the level, the more verbose the traces.
	26	*
	27	* It's possible to dynamically adjust level using variable g_debug_level,
	28	* which is only declared if DEBUGLEVEL>=2,
	29	* and is a global variable, not multi-thread protected (use with care)
	30	*/
	31
	32	#ifndef DEBUG_H_12987983217
	33	#define DEBUG_H_12987983217
	34
	35	#if defined (__cplusplus)
	36	extern "C" {
	37	#endif
	38
	39
	40	/* static assert is triggered at compile time, leaving no runtime artefact.
	41	* static assert only works with compile-time constants.
	42	* Also, this variant can only be used inside a function. */
	43	#define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1])
	44
	45
	46	/* DEBUGLEVEL is expected to be defined externally,
	47	* typically through compiler command line.
	48	* Value must be a number. */
	49	#ifndef DEBUGLEVEL
	50	# define DEBUGLEVEL 0
	51	#endif
	52
	53
	54	/* recommended values for DEBUGLEVEL :
	55	* 0 : release mode, no debug, all run-time checks disabled
	56	* 1 : enables assert() only, no display
	57	* 2 : reserved, for currently active debug path
	58	* 3 : events once per object lifetime (CCtx, CDict, etc.)
	59	* 4 : events once per frame
	60	* 5 : events once per block
	61	* 6 : events once per sequence (verbose)
	62	* 7+: events at every position (very verbose)
	63	*
	64	* It's generally inconvenient to output traces > 5.
	65	* In which case, it's possible to selectively trigger high verbosity levels
	66	* by modifying g_debug_level.
	67	*/
	68
	69	#if (DEBUGLEVEL>=1)
	70	# define ZSTD_DEPS_NEED_ASSERT
	71	# include "zstd_deps.h"
	72	#else
	73	# ifndef assert /* assert may be already defined, due to prior #include <assert.h> */
	74	# define assert(condition) ((void)0) /* disable assert (default) */
	75	# endif
	76	#endif
	77
	78	#if (DEBUGLEVEL>=2)
	79	# define ZSTD_DEPS_NEED_IO
	80	# include "zstd_deps.h"
	81	extern int g_debuglevel; /* the variable is only declared,
	82	it actually lives in debug.c,
	83	and is shared by the whole process.
	84	It's not thread-safe.
	85	It's useful when enabling very verbose levels
	86	on selective conditions (such as position in src) */
	87
	88	# define RAWLOG(l, ...) { \
	89	if (l<=g_debuglevel) { \
	90	ZSTD_DEBUG_PRINT(__VA_ARGS__); \
	91	} }
	92	# define DEBUGLOG(l, ...) { \
	93	if (l<=g_debuglevel) { \
	94	ZSTD_DEBUG_PRINT(__FILE__ ": " __VA_ARGS__); \
	95	ZSTD_DEBUG_PRINT(" \n"); \
	96	} }
	97	#else
	98	# define RAWLOG(l, ...) {} /* disabled */
	99	# define DEBUGLOG(l, ...) {} /* disabled */
	100	#endif
	101
	102
	103	#if defined (__cplusplus)
	104	}
	105	#endif
	106
	107	#endif /* DEBUG_H_12987983217 */

stage1/zstd/lib/common/entropy_common.c created+368

...	...	@@ -0,0 +1,368 @@
	1	/* ******************************************************************
	2	* Common functions of New Generation Entropy library
	3	* Copyright (c) Yann Collet, Facebook, Inc.
	4	*
	5	* You can contact the author at :
	6	* - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
	7	* - Public forum : https://groups.google.com/forum/#!forum/lz4c
	8	*
	9	* This source code is licensed under both the BSD-style license (found in the
	10	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	11	* in the COPYING file in the root directory of this source tree).
	12	* You may select, at your option, one of the above-listed licenses.
	13	****************************************************************** */
	14
	15	/* *************************************
	16	* Dependencies
	17	***************************************/
	18	#include "mem.h"
	19	#include "error_private.h" /* ERR_, ERROR /
	20	#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */
	21	#include "fse.h"
	22	#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */
	23	#include "huf.h"
	24
	25
	26	/=== Version ===/
	27	unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; }
	28
	29
	30	/=== Error Management ===/
	31	unsigned FSE_isError(size_t code) { return ERR_isError(code); }
	32	const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); }
	33
	34	unsigned HUF_isError(size_t code) { return ERR_isError(code); }
	35	const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
	36
	37
	38	/-*************************************************************
	39	* FSE NCount encoding-decoding
	40	****************************************************************/
	41	static U32 FSE_ctz(U32 val)
	42	{
	43	assert(val != 0);
	44	{
	45	# if defined(_MSC_VER) /* Visual */
	46	if (val != 0) {
	47	unsigned long r;
	48	_BitScanForward(&r, val);
	49	return (unsigned)r;
	50	} else {
	51	/* Should not reach this code path */
	52	__assume(0);
	53	}
	54	# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
	55	return __builtin_ctz(val);
	56	# elif defined(__ICCARM__) /* IAR Intrinsic */
	57	return __CTZ(val);
	58	# else /* Software version */
	59	U32 count = 0;
	60	while ((val & 1) == 0) {
	61	val >>= 1;
	62	++count;
	63	}
	64	return count;
	65	# endif
	66	}
	67	}
	68
	69	FORCE_INLINE_TEMPLATE
	70	size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
	71	const void* headerBuffer, size_t hbSize)
	72	{
	73	const BYTE* const istart = (const BYTE*) headerBuffer;
	74	const BYTE* const iend = istart + hbSize;
	75	const BYTE* ip = istart;
	76	int nbBits;
	77	int remaining;
	78	int threshold;
	79	U32 bitStream;
	80	int bitCount;
	81	unsigned charnum = 0;
	82	unsigned const maxSV1 = *maxSVPtr + 1;
	83	int previous0 = 0;
	84
	85	if (hbSize < 8) {
	86	/* This function only works when hbSize >= 8 */
	87	char buffer[8] = {0};
	88	ZSTD_memcpy(buffer, headerBuffer, hbSize);
	89	{ size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr,
	90	buffer, sizeof(buffer));
	91	if (FSE_isError(countSize)) return countSize;
	92	if (countSize > hbSize) return ERROR(corruption_detected);
	93	return countSize;
	94	} }
	95	assert(hbSize >= 8);
	96
	97	/* init */
	98	ZSTD_memset(normalizedCounter, 0, (maxSVPtr+1) sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */
	99	bitStream = MEM_readLE32(ip);
	100	nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */
	101	if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge);
	102	bitStream >>= 4;
	103	bitCount = 4;
	104	*tableLogPtr = nbBits;
	105	remaining = (1<<nbBits)+1;
	106	threshold = 1<<nbBits;
	107	nbBits++;
	108
	109	for (;;) {
	110	if (previous0) {
	111	/* Count the number of repeats. Each time the
	112	* 2-bit repeat code is 0b11 there is another
	113	* repeat.
	114	* Avoid UB by setting the high bit to 1.
	115	*/
	116	int repeats = FSE_ctz(~bitStream \| 0x80000000) >> 1;
	117	while (repeats >= 12) {
	118	charnum += 3 * 12;
	119	if (LIKELY(ip <= iend-7)) {
	120	ip += 3;
	121	} else {
	122	bitCount -= (int)(8 * (iend - 7 - ip));
	123	bitCount &= 31;
	124	ip = iend - 4;
	125	}
	126	bitStream = MEM_readLE32(ip) >> bitCount;
	127	repeats = FSE_ctz(~bitStream \| 0x80000000) >> 1;
	128	}
	129	charnum += 3 * repeats;
	130	bitStream >>= 2 * repeats;
	131	bitCount += 2 * repeats;
	132
	133	/* Add the final repeat which isn't 0b11. */
	134	assert((bitStream & 3) < 3);
	135	charnum += bitStream & 3;
	136	bitCount += 2;
	137
	138	/* This is an error, but break and return an error
	139	* at the end, because returning out of a loop makes
	140	* it harder for the compiler to optimize.
	141	*/
	142	if (charnum >= maxSV1) break;
	143
	144	/* We don't need to set the normalized count to 0
	145	* because we already memset the whole buffer to 0.
	146	*/
	147
	148	if (LIKELY(ip <= iend-7) \|\| (ip + (bitCount>>3) <= iend-4)) {
	149	assert((bitCount >> 3) <= 3); /* For first condition to work */
	150	ip += bitCount>>3;
	151	bitCount &= 7;
	152	} else {
	153	bitCount -= (int)(8 * (iend - 4 - ip));
	154	bitCount &= 31;
	155	ip = iend - 4;
	156	}
	157	bitStream = MEM_readLE32(ip) >> bitCount;
	158	}
	159	{
	160	int const max = (2*threshold-1) - remaining;
	161	int count;
	162
	163	if ((bitStream & (threshold-1)) < (U32)max) {
	164	count = bitStream & (threshold-1);
	165	bitCount += nbBits-1;
	166	} else {
	167	count = bitStream & (2*threshold-1);
	168	if (count >= threshold) count -= max;
	169	bitCount += nbBits;
	170	}
	171
	172	count--; /* extra accuracy */
	173	/* When it matters (small blocks), this is a
	174	* predictable branch, because we don't use -1.
	175	*/
	176	if (count >= 0) {
	177	remaining -= count;
	178	} else {
	179	assert(count == -1);
	180	remaining += count;
	181	}
	182	normalizedCounter[charnum++] = (short)count;
	183	previous0 = !count;
	184
	185	assert(threshold > 1);
	186	if (remaining < threshold) {
	187	/* This branch can be folded into the
	188	* threshold update condition because we
	189	* know that threshold > 1.
	190	*/
	191	if (remaining <= 1) break;
	192	nbBits = BIT_highbit32(remaining) + 1;
	193	threshold = 1 << (nbBits - 1);
	194	}
	195	if (charnum >= maxSV1) break;
	196
	197	if (LIKELY(ip <= iend-7) \|\| (ip + (bitCount>>3) <= iend-4)) {
	198	ip += bitCount>>3;
	199	bitCount &= 7;
	200	} else {
	201	bitCount -= (int)(8 * (iend - 4 - ip));
	202	bitCount &= 31;
	203	ip = iend - 4;
	204	}
	205	bitStream = MEM_readLE32(ip) >> bitCount;
	206	} }
	207	if (remaining != 1) return ERROR(corruption_detected);
	208	/* Only possible when there are too many zeros. */
	209	if (charnum > maxSV1) return ERROR(maxSymbolValue_tooSmall);
	210	if (bitCount > 32) return ERROR(corruption_detected);
	211	*maxSVPtr = charnum-1;
	212
	213	ip += (bitCount+7)>>3;
	214	return ip-istart;
	215	}
	216
	217	/* Avoids the FORCE_INLINE of the _body() function. */
	218	static size_t FSE_readNCount_body_default(
	219	short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
	220	const void* headerBuffer, size_t hbSize)
	221	{
	222	return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
	223	}
	224
	225	#if DYNAMIC_BMI2
	226	BMI2_TARGET_ATTRIBUTE static size_t FSE_readNCount_body_bmi2(
	227	short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
	228	const void* headerBuffer, size_t hbSize)
	229	{
	230	return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
	231	}
	232	#endif
	233
	234	size_t FSE_readNCount_bmi2(
	235	short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
	236	const void* headerBuffer, size_t hbSize, int bmi2)
	237	{
	238	#if DYNAMIC_BMI2
	239	if (bmi2) {
	240	return FSE_readNCount_body_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
	241	}
	242	#endif
	243	(void)bmi2;
	244	return FSE_readNCount_body_default(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
	245	}
	246
	247	size_t FSE_readNCount(
	248	short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
	249	const void* headerBuffer, size_t hbSize)
	250	{
	251	return FSE_readNCount_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize, /* bmi2 */ 0);
	252	}
	253
	254
	255	/*! HUF_readStats() :
	256	Read compact Huffman tree, saved by HUF_writeCTable().
	257	`huffWeight` is destination buffer.
	258	`rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32.
	259	@return : size read from `src` , or an error Code .
	260	Note : Needed by HUF_readCTable() and HUF_readDTableX?() .
	261	*/
	262	size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
	263	U32* nbSymbolsPtr, U32* tableLogPtr,
	264	const void* src, size_t srcSize)
	265	{
	266	U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
	267	return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0);
	268	}
	269
	270	FORCE_INLINE_TEMPLATE size_t
	271	HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats,
	272	U32* nbSymbolsPtr, U32* tableLogPtr,
	273	const void* src, size_t srcSize,
	274	void* workSpace, size_t wkspSize,
	275	int bmi2)
	276	{
	277	U32 weightTotal;
	278	const BYTE* ip = (const BYTE*) src;
	279	size_t iSize;
	280	size_t oSize;
	281
	282	if (!srcSize) return ERROR(srcSize_wrong);
	283	iSize = ip[0];
	284	/* ZSTD_memset(huffWeight, 0, hwSize); // is not necessary, even though some analyzer complain ... */
	285
	286	if (iSize >= 128) { /* special header */
	287	oSize = iSize - 127;
	288	iSize = ((oSize+1)/2);
	289	if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
	290	if (oSize >= hwSize) return ERROR(corruption_detected);
	291	ip += 1;
	292	{ U32 n;
	293	for (n=0; n<oSize; n+=2) {
	294	huffWeight[n] = ip[n/2] >> 4;
	295	huffWeight[n+1] = ip[n/2] & 15;
	296	} } }
	297	else { /* header compressed with FSE (normal case) */
	298	if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
	299	/* max (hwSize-1) values decoded, as last one is implied */
	300	oSize = FSE_decompress_wksp_bmi2(huffWeight, hwSize-1, ip+1, iSize, 6, workSpace, wkspSize, bmi2);
	301	if (FSE_isError(oSize)) return oSize;
	302	}
	303
	304	/* collect weight stats */
	305	ZSTD_memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32));
	306	weightTotal = 0;
	307	{ U32 n; for (n=0; n<oSize; n++) {
	308	if (huffWeight[n] > HUF_TABLELOG_MAX) return ERROR(corruption_detected);
	309	rankStats[huffWeight[n]]++;
	310	weightTotal += (1 << huffWeight[n]) >> 1;
	311	} }
	312	if (weightTotal == 0) return ERROR(corruption_detected);
	313
	314	/* get last non-null symbol weight (implied, total must be 2^n) */
	315	{ U32 const tableLog = BIT_highbit32(weightTotal) + 1;
	316	if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected);
	317	*tableLogPtr = tableLog;
	318	/* determine last weight */
	319	{ U32 const total = 1 << tableLog;
	320	U32 const rest = total - weightTotal;
	321	U32 const verif = 1 << BIT_highbit32(rest);
	322	U32 const lastWeight = BIT_highbit32(rest) + 1;
	323	if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
	324	huffWeight[oSize] = (BYTE)lastWeight;
	325	rankStats[lastWeight]++;
	326	} }
	327
	328	/* check tree construction validity */
	329	if ((rankStats[1] < 2) \|\| (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
	330
	331	/* results */
	332	*nbSymbolsPtr = (U32)(oSize+1);
	333	return iSize+1;
	334	}
	335
	336	/* Avoids the FORCE_INLINE of the _body() function. */
	337	static size_t HUF_readStats_body_default(BYTE* huffWeight, size_t hwSize, U32* rankStats,
	338	U32* nbSymbolsPtr, U32* tableLogPtr,
	339	const void* src, size_t srcSize,
	340	void* workSpace, size_t wkspSize)
	341	{
	342	return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 0);
	343	}
	344
	345	#if DYNAMIC_BMI2
	346	static BMI2_TARGET_ATTRIBUTE size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats,
	347	U32* nbSymbolsPtr, U32* tableLogPtr,
	348	const void* src, size_t srcSize,
	349	void* workSpace, size_t wkspSize)
	350	{
	351	return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 1);
	352	}
	353	#endif
	354
	355	size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats,
	356	U32* nbSymbolsPtr, U32* tableLogPtr,
	357	const void* src, size_t srcSize,
	358	void* workSpace, size_t wkspSize,
	359	int bmi2)
	360	{
	361	#if DYNAMIC_BMI2
	362	if (bmi2) {
	363	return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
	364	}
	365	#endif
	366	(void)bmi2;
	367	return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
	368	}

stage1/zstd/lib/common/error_private.c created+56

...	...	@@ -0,0 +1,56 @@
	1	/*
	2	* Copyright (c) Yann Collet, Facebook, Inc.
	3	* All rights reserved.
	4	*
	5	* This source code is licensed under both the BSD-style license (found in the
	6	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	7	* in the COPYING file in the root directory of this source tree).
	8	* You may select, at your option, one of the above-listed licenses.
	9	*/
	10
	11	/* The purpose of this file is to have a single list of error strings embedded in binary */
	12
	13	#include "error_private.h"
	14
	15	const char* ERR_getErrorString(ERR_enum code)
	16	{
	17	#ifdef ZSTD_STRIP_ERROR_STRINGS
	18	(void)code;
	19	return "Error strings stripped";
	20	#else
	21	static const char* const notErrorCode = "Unspecified error code";
	22	switch( code )
	23	{
	24	case PREFIX(no_error): return "No error detected";
	25	case PREFIX(GENERIC): return "Error (generic)";
	26	case PREFIX(prefix_unknown): return "Unknown frame descriptor";
	27	case PREFIX(version_unsupported): return "Version not supported";
	28	case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
	29	case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
	30	case PREFIX(corruption_detected): return "Corrupted block detected";
	31	case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
	32	case PREFIX(parameter_unsupported): return "Unsupported parameter";
	33	case PREFIX(parameter_outOfBound): return "Parameter is out of bound";
	34	case PREFIX(init_missing): return "Context should be init first";
	35	case PREFIX(memory_allocation): return "Allocation error : not enough memory";
	36	case PREFIX(workSpace_tooSmall): return "workSpace buffer is not large enough";
	37	case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
	38	case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
	39	case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
	40	case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
	41	case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
	42	case PREFIX(dictionary_wrong): return "Dictionary mismatch";
	43	case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
	44	case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
	45	case PREFIX(srcSize_wrong): return "Src size is incorrect";
	46	case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer";
	47	/* following error codes are not stable and may be removed or changed in a future version */
	48	case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
	49	case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
	50	case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong";
	51	case PREFIX(srcBuffer_wrong): return "Source buffer is wrong";
	52	case PREFIX(maxCode):
	53	default: return notErrorCode;
	54	}
	55	#endif
	56	}

stage1/zstd/lib/common/error_private.h created+159

...	...	@@ -0,0 +1,159 @@
	1	/*
	2	* Copyright (c) Yann Collet, Facebook, Inc.
	3	* All rights reserved.
	4	*
	5	* This source code is licensed under both the BSD-style license (found in the
	6	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	7	* in the COPYING file in the root directory of this source tree).
	8	* You may select, at your option, one of the above-listed licenses.
	9	*/
	10
	11	/* Note : this module is expected to remain private, do not expose it */
	12
	13	#ifndef ERROR_H_MODULE
	14	#define ERROR_H_MODULE
	15
	16	#if defined (__cplusplus)
	17	extern "C" {
	18	#endif
	19
	20
	21	/* ****************************************
	22	* Dependencies
	23	******************************************/
	24	#include "../zstd_errors.h" /* enum list */
	25	#include "compiler.h"
	26	#include "debug.h"
	27	#include "zstd_deps.h" /* size_t */
	28
	29
	30	/* ****************************************
	31	* Compiler-specific
	32	******************************************/
	33	#if defined(__GNUC__)
	34	# define ERR_STATIC static __attribute__((unused))
	35	#elif defined (__cplusplus) \|\| (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
	36	# define ERR_STATIC static inline
	37	#elif defined(_MSC_VER)
	38	# define ERR_STATIC static __inline
	39	#else
	40	# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
	41	#endif
	42
	43
	44	/-***************************************
	45	* Customization (error_public.h)
	46	******************************************/
	47	typedef ZSTD_ErrorCode ERR_enum;
	48	#define PREFIX(name) ZSTD_error_##name
	49
	50
	51	/-***************************************
	52	* Error codes handling
	53	******************************************/
	54	#undef ERROR /* already defined on Visual Studio */
	55	#define ERROR(name) ZSTD_ERROR(name)
	56	#define ZSTD_ERROR(name) ((size_t)-PREFIX(name))
	57
	58	ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
	59
	60	ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
	61
	62	/* check and forward error code */
	63	#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e
	64	#define CHECK_F(f) { CHECK_V_F(_var_err__, f); }
	65
	66
	67	/-***************************************
	68	* Error Strings
	69	******************************************/
	70
	71	const char* ERR_getErrorString(ERR_enum code); /* error_private.c */
	72
	73	ERR_STATIC const char* ERR_getErrorName(size_t code)
	74	{
	75	return ERR_getErrorString(ERR_getErrorCode(code));
	76	}
	77
	78	/**
	79	* Ignore: this is an internal helper.
	80	*
	81	* This is a helper function to help force C99-correctness during compilation.
	82	* Under strict compilation modes, variadic macro arguments can't be empty.
	83	* However, variadic function arguments can be. Using a function therefore lets
	84	* us statically check that at least one (string) argument was passed,
	85	* independent of the compilation flags.
	86	*/
	87	static INLINE_KEYWORD UNUSED_ATTR
	88	void _force_has_format_string(const char *format, ...) {
	89	(void)format;
	90	}
	91
	92	/**
	93	* Ignore: this is an internal helper.
	94	*
	95	* We want to force this function invocation to be syntactically correct, but
	96	* we don't want to force runtime evaluation of its arguments.
	97	*/
	98	#define _FORCE_HAS_FORMAT_STRING(...) \
	99	if (0) { \
	100	_force_has_format_string(__VA_ARGS__); \
	101	}
	102
	103	#define ERR_QUOTE(str) #str
	104
	105	/**
	106	* Return the specified error if the condition evaluates to true.
	107	*
	108	* In debug modes, prints additional information.
	109	* In order to do that (particularly, printing the conditional that failed),
	110	* this can't just wrap RETURN_ERROR().
	111	*/
	112	#define RETURN_ERROR_IF(cond, err, ...) \
	113	if (cond) { \
	114	RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \
	115	__FILE__, __LINE__, ERR_QUOTE(cond), ERR_QUOTE(ERROR(err))); \
	116	_FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
	117	RAWLOG(3, ": " __VA_ARGS__); \
	118	RAWLOG(3, "\n"); \
	119	return ERROR(err); \
	120	}
	121
	122	/**
	123	* Unconditionally return the specified error.
	124	*
	125	* In debug modes, prints additional information.
	126	*/
	127	#define RETURN_ERROR(err, ...) \
	128	do { \
	129	RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \
	130	__FILE__, __LINE__, ERR_QUOTE(ERROR(err))); \
	131	_FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
	132	RAWLOG(3, ": " __VA_ARGS__); \
	133	RAWLOG(3, "\n"); \
	134	return ERROR(err); \
	135	} while(0);
	136
	137	/**
	138	* If the provided expression evaluates to an error code, returns that error code.
	139	*
	140	* In debug modes, prints additional information.
	141	*/
	142	#define FORWARD_IF_ERROR(err, ...) \
	143	do { \
	144	size_t const err_code = (err); \
	145	if (ERR_isError(err_code)) { \
	146	RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \
	147	__FILE__, __LINE__, ERR_QUOTE(err), ERR_getErrorName(err_code)); \
	148	_FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \
	149	RAWLOG(3, ": " __VA_ARGS__); \
	150	RAWLOG(3, "\n"); \
	151	return err_code; \
	152	} \
	153	} while(0);
	154
	155	#if defined (__cplusplus)
	156	}
	157	#endif
	158
	159	#endif /* ERROR_H_MODULE */

stage1/zstd/lib/common/fse.h created+717

...	...	@@ -0,0 +1,717 @@
	1	/* ******************************************************************
	2	* FSE : Finite State Entropy codec
	3	* Public Prototypes declaration
	4	* Copyright (c) Yann Collet, Facebook, Inc.
	5	*
	6	* You can contact the author at :
	7	* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
	8	*
	9	* This source code is licensed under both the BSD-style license (found in the
	10	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	11	* in the COPYING file in the root directory of this source tree).
	12	* You may select, at your option, one of the above-listed licenses.
	13	****************************************************************** */
	14
	15	#if defined (__cplusplus)
	16	extern "C" {
	17	#endif
	18
	19	#ifndef FSE_H
	20	#define FSE_H
	21
	22
	23	/-****************************************
	24	* Dependencies
	25	******************************************/
	26	#include "zstd_deps.h" /* size_t, ptrdiff_t */
	27
	28
	29	/-****************************************
	30	* FSE_PUBLIC_API : control library symbols visibility
	31	******************************************/
	32	#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
	33	# define FSE_PUBLIC_API __attribute__ ((visibility ("default")))
	34	#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */
	35	# define FSE_PUBLIC_API __declspec(dllexport)
	36	#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
	37	# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
	38	#else
	39	# define FSE_PUBLIC_API
	40	#endif
	41
	42	/------ Version ------/
	43	#define FSE_VERSION_MAJOR 0
	44	#define FSE_VERSION_MINOR 9
	45	#define FSE_VERSION_RELEASE 0
	46
	47	#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE
	48	#define FSE_QUOTE(str) #str
	49	#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str)
	50	#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION)
	51
	52	#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR 100100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE)
	53	FSE_PUBLIC_API unsigned FSE_versionNumber(void); /*< library version number; to be used when checking dll version /
	54
	55
	56	/-***************************************
	57	* FSE simple functions
	58	******************************************/
	59	/*! FSE_compress() :
	60	Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
	61	'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize).
	62	@return : size of compressed data (<= dstCapacity).
	63	Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
	64	if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
	65	if FSE_isError(return), compression failed (more details using FSE_getErrorName())
	66	*/
	67	FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity,
	68	const void* src, size_t srcSize);
	69
	70	/*! FSE_decompress():
	71	Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
	72	into already allocated destination buffer 'dst', of size 'dstCapacity'.
	73	@return : size of regenerated data (<= maxDstSize),
	74	or an error code, which can be tested using FSE_isError() .
	75
	76	Important : FSE_decompress() does not decompress non-compressible nor RLE data !!!
	77	Why ? : making this distinction requires a header.
	78	Header management is intentionally delegated to the user layer, which can better manage special cases.
	79	*/
	80	FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity,
	81	const void* cSrc, size_t cSrcSize);
	82
	83
	84	/-****************************************
	85	* Tool functions
	86	******************************************/
	87	FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */
	88
	89	/* Error Management */
	90	FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */
	91	FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */
	92
	93
	94	/-****************************************
	95	* FSE advanced functions
	96	******************************************/
	97	/*! FSE_compress2() :
	98	Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog'
	99	Both parameters can be defined as '0' to mean : use default value
	100	@return : size of compressed data
	101	Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!
	102	if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
	103	if FSE_isError(return), it's an error code.
	104	*/
	105	FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
	106
	107
	108	/-****************************************
	109	* FSE detailed API
	110	******************************************/
	111	/*!
	112	FSE_compress() does the following:
	113	1. count symbol occurrence from source[] into table count[] (see hist.h)
	114	2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
	115	3. save normalized counters to memory buffer using writeNCount()
	116	4. build encoding table 'CTable' from normalized counters
	117	5. encode the data stream using encoding table 'CTable'
	118
	119	FSE_decompress() does the following:
	120	1. read normalized counters with readNCount()
	121	2. build decoding table 'DTable' from normalized counters
	122	3. decode the data stream using decoding table 'DTable'
	123
	124	The following API allows targeting specific sub-functions for advanced tasks.
	125	For example, it's possible to compress several blocks using the same 'CTable',
	126	or to save and provide normalized distribution using external method.
	127	*/
	128
	129	/* * COMPRESSION * */
	130
	131	/*! FSE_optimalTableLog():
	132	dynamically downsize 'tableLog' when conditions are met.
	133	It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
	134	@return : recommended tableLog (necessarily <= 'maxTableLog') */
	135	FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
	136
	137	/*! FSE_normalizeCount():
	138	normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)
	139	'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
	140	useLowProbCount is a boolean parameter which trades off compressed size for
	141	faster header decoding. When it is set to 1, the compressed data will be slightly
	142	smaller. And when it is set to 0, FSE_readNCount() and FSE_buildDTable() will be
	143	faster. If you are compressing a small amount of data (< 2 KB) then useLowProbCount=0
	144	is a good default, since header deserialization makes a big speed difference.
	145	Otherwise, useLowProbCount=1 is a good default, since the speed difference is small.
	146	@return : tableLog,
	147	or an errorCode, which can be tested using FSE_isError() */
	148	FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog,
	149	const unsigned* count, size_t srcSize, unsigned maxSymbolValue, unsigned useLowProbCount);
	150
	151	/*! FSE_NCountWriteBound():
	152	Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
	153	Typically useful for allocation purpose. */
	154	FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
	155
	156	/*! FSE_writeNCount():
	157	Compactly save 'normalizedCounter' into 'buffer'.
	158	@return : size of the compressed table,
	159	or an errorCode, which can be tested using FSE_isError(). */
	160	FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize,
	161	const short* normalizedCounter,
	162	unsigned maxSymbolValue, unsigned tableLog);
	163
	164	/*! Constructor and Destructor of FSE_CTable.
	165	Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
	166	typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */
	167	FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog);
	168	FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct);
	169
	170	/*! FSE_buildCTable():
	171	Builds `ct`, which must be already allocated, using FSE_createCTable().
	172	@return : 0, or an errorCode, which can be tested using FSE_isError() */
	173	FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
	174
	175	/*! FSE_compress_usingCTable():
	176	Compress `src` using `ct` into `dst` which must be already allocated.
	177	@return : size of compressed data (<= `dstCapacity`),
	178	or 0 if compressed data could not fit into `dst`,
	179	or an errorCode, which can be tested using FSE_isError() */
	180	FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct);
	181
	182	/*!
	183	Tutorial :
	184	----------
	185	The first step is to count all symbols. FSE_count() does this job very fast.
	186	Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells.
	187	'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0]
	188	maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value)
	189	FSE_count() will return the number of occurrence of the most frequent symbol.
	190	This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility.
	191	If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
	192
	193	The next step is to normalize the frequencies.
	194	FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'.
	195	It also guarantees a minimum of 1 to any Symbol with frequency >= 1.
	196	You can use 'tableLog'==0 to mean "use default tableLog value".
	197	If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(),
	198	which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default").
	199
	200	The result of FSE_normalizeCount() will be saved into a table,
	201	called 'normalizedCounter', which is a table of signed short.
	202	'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells.
	203	The return value is tableLog if everything proceeded as expected.
	204	It is 0 if there is a single symbol within distribution.
	205	If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).
	206
	207	'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount().
	208	'buffer' must be already allocated.
	209	For guaranteed success, buffer size must be at least FSE_headerBound().
	210	The result of the function is the number of bytes written into 'buffer'.
	211	If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small).
	212
	213	'normalizedCounter' can then be used to create the compression table 'CTable'.
	214	The space required by 'CTable' must be already allocated, using FSE_createCTable().
	215	You can then use FSE_buildCTable() to fill 'CTable'.
	216	If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()).
	217
	218	'CTable' can then be used to compress 'src', with FSE_compress_usingCTable().
	219	Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize'
	220	The function returns the size of compressed data (without header), necessarily <= `dstCapacity`.
	221	If it returns '0', compressed data could not fit into 'dst'.
	222	If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
	223	*/
	224
	225
	226	/* * DECOMPRESSION * */
	227
	228	/*! FSE_readNCount():
	229	Read compactly saved 'normalizedCounter' from 'rBuffer'.
	230	@return : size read from 'rBuffer',
	231	or an errorCode, which can be tested using FSE_isError().
	232	maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
	233	FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter,
	234	unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
	235	const void* rBuffer, size_t rBuffSize);
	236
	237	/*! FSE_readNCount_bmi2():
	238	* Same as FSE_readNCount() but pass bmi2=1 when your CPU supports BMI2 and 0 otherwise.
	239	*/
	240	FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter,
	241	unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
	242	const void* rBuffer, size_t rBuffSize, int bmi2);
	243
	244	/*! Constructor and Destructor of FSE_DTable.
	245	Note that its size depends on 'tableLog' */
	246	typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
	247	FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog);
	248	FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt);
	249
	250	/*! FSE_buildDTable():
	251	Builds 'dt', which must be already allocated, using FSE_createDTable().
	252	return : 0, or an errorCode, which can be tested using FSE_isError() */
	253	FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
	254
	255	/*! FSE_decompress_usingDTable():
	256	Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
	257	into `dst` which must be already allocated.
	258	@return : size of regenerated data (necessarily <= `dstCapacity`),
	259	or an errorCode, which can be tested using FSE_isError() */
	260	FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
	261
	262	/*!
	263	Tutorial :
	264	----------
	265	(Note : these functions only decompress FSE-compressed blocks.
	266	If block is uncompressed, use memcpy() instead
	267	If block is a single repeated byte, use memset() instead )
	268
	269	The first step is to obtain the normalized frequencies of symbols.
	270	This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount().
	271	'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short.
	272	In practice, that means it's necessary to know 'maxSymbolValue' beforehand,
	273	or size the table to handle worst case situations (typically 256).
	274	FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'.
	275	The result of FSE_readNCount() is the number of bytes read from 'rBuffer'.
	276	Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that.
	277	If there is an error, the function will return an error code, which can be tested using FSE_isError().
	278
	279	The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'.
	280	This is performed by the function FSE_buildDTable().
	281	The space required by 'FSE_DTable' must be already allocated using FSE_createDTable().
	282	If there is an error, the function will return an error code, which can be tested using FSE_isError().
	283
	284	`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable().
	285	`cSrcSize` must be strictly correct, otherwise decompression will fail.
	286	FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`).
	287	If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small)
	288	*/
	289
	290	#endif /* FSE_H */
	291
	292	#if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY)
	293	#define FSE_H_FSE_STATIC_LINKING_ONLY
	294
	295	/* * Dependency * */
	296	#include "bitstream.h"
	297
	298
	299	/* *****************************************
	300	* Static allocation
	301	*******************************************/
	302	/* FSE buffer bounds */
	303	#define FSE_NCOUNTBOUND 512
	304	#define FSE_BLOCKBOUND(size) ((size) + ((size)>>7) + 4 /* fse states / + sizeof(size_t) / bitContainer */)
	305	#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
	306
	307	/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */
	308	#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<((maxTableLog)-1)) + (((maxSymbolValue)+1)*2))
	309	#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<(maxTableLog)))
	310
	311	/* or use the size to malloc() space directly. Pay attention to alignment restrictions though */
	312	#define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue) (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable))
	313	#define FSE_DTABLE_SIZE(maxTableLog) (FSE_DTABLE_SIZE_U32(maxTableLog) * sizeof(FSE_DTable))
	314
	315
	316	/* *****************************************
	317	* FSE advanced API
	318	***************************************** */
	319
	320	unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
	321	/*< same as FSE_optimalTableLog(), which used `minus==2` /
	322
	323	/* FSE_compress_wksp() :
	324	* Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
	325	* FSE_COMPRESS_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable.
	326	*/
	327	#define FSE_COMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) )
	328	size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
	329
	330	size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
	331	/*< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits /
	332
	333	size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
	334	/*< build a fake FSE_CTable, designed to compress always the same symbolValue /
	335
	336	/* FSE_buildCTable_wksp() :
	337	* Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
	338	* `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`.
	339	* See FSE_buildCTable_wksp() for breakdown of workspace usage.
	340	*/
	341	#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (((maxSymbolValue + 2) + (1ull << (tableLog)))/2 + sizeof(U64)/sizeof(U32) /* additional 8 bytes for potential table overwrite */)
	342	#define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog))
	343	size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
	344
	345	#define FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) (sizeof(short) * (maxSymbolValue + 1) + (1ULL << maxTableLog) + 8)
	346	#define FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ((FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) + sizeof(unsigned) - 1) / sizeof(unsigned))
	347	FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
	348	/*< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` /
	349
	350	size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
	351	/*< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits /
	352
	353	size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
	354	/*< build a fake FSE_DTable, designed to always generate the same symbolValue /
	355
	356	#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1)
	357	#define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned))
	358	size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize);
	359	/*< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)` /
	360
	361	size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2);
	362	/*< Same as FSE_decompress_wksp() but with dynamic BMI2 support. Pass 1 if your CPU supports BMI2 or 0 if it doesn't. /
	363
	364	typedef enum {
	365	FSE_repeat_none, /*< Cannot use the previous table /
	366	FSE_repeat_check, /*< Can use the previous table but it must be checked /
	367	FSE_repeat_valid /*< Can use the previous table and it is assumed to be valid /
	368	} FSE_repeat;
	369
	370	/* *****************************************
	371	* FSE symbol compression API
	372	*******************************************/
	373	/*!
	374	This API consists of small unitary functions, which highly benefit from being inlined.
	375	Hence their body are included in next section.
	376	*/
	377	typedef struct {
	378	ptrdiff_t value;
	379	const void* stateTable;
	380	const void* symbolTT;
	381	unsigned stateLog;
	382	} FSE_CState_t;
	383
	384	static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct);
	385
	386	static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol);
	387
	388	static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr);
	389
	390	/**<
	391	These functions are inner components of FSE_compress_usingCTable().
	392	They allow the creation of custom streams, mixing multiple tables and bit sources.
	393
	394	A key property to keep in mind is that encoding and decoding are done in reverse direction.
	395	So the first symbol you will encode is the last you will decode, like a LIFO stack.
	396
	397	You will need a few variables to track your CStream. They are :
	398
	399	FSE_CTable ct; // Provided by FSE_buildCTable()
	400	BIT_CStream_t bitStream; // bitStream tracking structure
	401	FSE_CState_t state; // State tracking structure (can have several)
	402
	403
	404	The first thing to do is to init bitStream and state.
	405	size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize);
	406	FSE_initCState(&state, ct);
	407
	408	Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
	409	You can then encode your input data, byte after byte.
	410	FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
	411	Remember decoding will be done in reverse direction.
	412	FSE_encodeByte(&bitStream, &state, symbol);
	413
	414	At any time, you can also add any bit sequence.
	415	Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders
	416	BIT_addBits(&bitStream, bitField, nbBits);
	417
	418	The above methods don't commit data to memory, they just store it into local register, for speed.
	419	Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
	420	Writing data to memory is a manual operation, performed by the flushBits function.
	421	BIT_flushBits(&bitStream);
	422
	423	Your last FSE encoding operation shall be to flush your last state value(s).
	424	FSE_flushState(&bitStream, &state);
	425
	426	Finally, you must close the bitStream.
	427	The function returns the size of CStream in bytes.
	428	If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
	429	If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
	430	size_t size = BIT_closeCStream(&bitStream);
	431	*/
	432
	433
	434	/* *****************************************
	435	* FSE symbol decompression API
	436	*******************************************/
	437	typedef struct {
	438	size_t state;
	439	const void* table; /* precise table may vary, depending on U16 */
	440	} FSE_DState_t;
	441
	442
	443	static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt);
	444
	445	static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
	446
	447	static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
	448
	449	/**<
	450	Let's now decompose FSE_decompress_usingDTable() into its unitary components.
	451	You will decode FSE-encoded symbols from the bitStream,
	452	and also any other bitFields you put in, in reverse order.
	453
	454	You will need a few variables to track your bitStream. They are :
	455
	456	BIT_DStream_t DStream; // Stream context
	457	FSE_DState_t DState; // State context. Multiple ones are possible
	458	FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
	459
	460	The first thing to do is to init the bitStream.
	461	errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
	462
	463	You should then retrieve your initial state(s)
	464	(in reverse flushing order if you have several ones) :
	465	errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
	466
	467	You can then decode your data, symbol after symbol.
	468	For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
	469	Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
	470	unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
	471
	472	You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
	473	Note : maximum allowed nbBits is 25, for 32-bits compatibility
	474	size_t bitField = BIT_readBits(&DStream, nbBits);
	475
	476	All above operations only read from local register (which size depends on size_t).
	477	Refueling the register from memory is manually performed by the reload method.
	478	endSignal = FSE_reloadDStream(&DStream);
	479
	480	BIT_reloadDStream() result tells if there is still some more data to read from DStream.
	481	BIT_DStream_unfinished : there is still some data left into the DStream.
	482	BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
	483	BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
	484	BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
	485
	486	When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
	487	to properly detect the exact end of stream.
	488	After each decoded symbol, check if DStream is fully consumed using this simple test :
	489	BIT_reloadDStream(&DStream) >= BIT_DStream_completed
	490
	491	When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
	492	Checking if DStream has reached its end is performed by :
	493	BIT_endOfDStream(&DStream);
	494	Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
	495	FSE_endOfDState(&DState);
	496	*/
	497
	498
	499	/* *****************************************
	500	* FSE unsafe API
	501	*******************************************/
	502	static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
	503	/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
	504
	505
	506	/* *****************************************
	507	* Implementation of inlined functions
	508	*******************************************/
	509	typedef struct {
	510	int deltaFindState;
	511	U32 deltaNbBits;
	512	} FSE_symbolCompressionTransform; /* total 8 bytes */
	513
	514	MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
	515	{
	516	const void* ptr = ct;
	517	const U16* u16ptr = (const U16*) ptr;
	518	const U32 tableLog = MEM_read16(ptr);
	519	statePtr->value = (ptrdiff_t)1<<tableLog;
	520	statePtr->stateTable = u16ptr+2;
	521	statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1);
	522	statePtr->stateLog = tableLog;
	523	}
	524
	525
	526	/*! FSE_initCState2() :
	527	* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read)
	528	* uses the smallest state value possible, saving the cost of this symbol */
	529	MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol)
	530	{
	531	FSE_initCState(statePtr, ct);
	532	{ const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
	533	const U16* stateTable = (const U16*)(statePtr->stateTable);
	534	U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16);
	535	statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits;
	536	statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
	537	}
	538	}
	539
	540	MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol)
	541	{
	542	FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
	543	const U16* const stateTable = (const U16*)(statePtr->stateTable);
	544	U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
	545	BIT_addBits(bitC, statePtr->value, nbBitsOut);
	546	statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
	547	}
	548
	549	MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr)
	550	{
	551	BIT_addBits(bitC, statePtr->value, statePtr->stateLog);
	552	BIT_flushBits(bitC);
	553	}
	554
	555
	556	/* FSE_getMaxNbBits() :
	557	* Approximate maximum cost of a symbol, in bits.
	558	* Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2)
	559	* note 1 : assume symbolValue is valid (<= maxSymbolValue)
	560	* note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
	561	MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue)
	562	{
	563	const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
	564	return (symbolTT[symbolValue].deltaNbBits + ((1<<16)-1)) >> 16;
	565	}
	566
	567	/* FSE_bitCost() :
	568	* Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits)
	569	* note 1 : assume symbolValue is valid (<= maxSymbolValue)
	570	* note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
	571	MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog)
	572	{
	573	const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr;
	574	U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16;
	575	U32 const threshold = (minNbBits+1) << 16;
	576	assert(tableLog < 16);
	577	assert(accuracyLog < 31-tableLog); /* ensure enough room for renormalization double shift */
	578	{ U32 const tableSize = 1 << tableLog;
	579	U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize);
	580	U32 const normalizedDeltaFromThreshold = (deltaFromThreshold << accuracyLog) >> tableLog; /* linear interpolation (very approximate) */
	581	U32 const bitMultiplier = 1 << accuracyLog;
	582	assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold);
	583	assert(normalizedDeltaFromThreshold <= bitMultiplier);
	584	return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold;
	585	}
	586	}
	587
	588
	589	/* ====== Decompression ====== */
	590
	591	typedef struct {
	592	U16 tableLog;
	593	U16 fastMode;
	594	} FSE_DTableHeader; /* sizeof U32 */
	595
	596	typedef struct
	597	{
	598	unsigned short newState;
	599	unsigned char symbol;
	600	unsigned char nbBits;
	601	} FSE_decode_t; /* size == U32 */
	602
	603	MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt)
	604	{
	605	const void* ptr = dt;
	606	const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr;
	607	DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
	608	BIT_reloadDStream(bitD);
	609	DStatePtr->table = dt + 1;
	610	}
	611
	612	MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr)
	613	{
	614	FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
	615	return DInfo.symbol;
	616	}
	617
	618	MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
	619	{
	620	FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
	621	U32 const nbBits = DInfo.nbBits;
	622	size_t const lowBits = BIT_readBits(bitD, nbBits);
	623	DStatePtr->state = DInfo.newState + lowBits;
	624	}
	625
	626	MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
	627	{
	628	FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
	629	U32 const nbBits = DInfo.nbBits;
	630	BYTE const symbol = DInfo.symbol;
	631	size_t const lowBits = BIT_readBits(bitD, nbBits);
	632
	633	DStatePtr->state = DInfo.newState + lowBits;
	634	return symbol;
	635	}
	636
	637	/*! FSE_decodeSymbolFast() :
	638	unsafe, only works if no symbol has a probability > 50% */
	639	MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
	640	{
	641	FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
	642	U32 const nbBits = DInfo.nbBits;
	643	BYTE const symbol = DInfo.symbol;
	644	size_t const lowBits = BIT_readBitsFast(bitD, nbBits);
	645
	646	DStatePtr->state = DInfo.newState + lowBits;
	647	return symbol;
	648	}
	649
	650	MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
	651	{
	652	return DStatePtr->state == 0;
	653	}
	654
	655
	656
	657	#ifndef FSE_COMMONDEFS_ONLY
	658
	659	/* **************************************************************
	660	* Tuning parameters
	661	****************************************************************/
	662	/*!MEMORY_USAGE :
	663	* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
	664	* Increasing memory usage improves compression ratio
	665	* Reduced memory usage can improve speed, due to cache effect
	666	* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
	667	#ifndef FSE_MAX_MEMORY_USAGE
	668	# define FSE_MAX_MEMORY_USAGE 14
	669	#endif
	670	#ifndef FSE_DEFAULT_MEMORY_USAGE
	671	# define FSE_DEFAULT_MEMORY_USAGE 13
	672	#endif
	673	#if (FSE_DEFAULT_MEMORY_USAGE > FSE_MAX_MEMORY_USAGE)
	674	# error "FSE_DEFAULT_MEMORY_USAGE must be <= FSE_MAX_MEMORY_USAGE"
	675	#endif
	676
	677	/*!FSE_MAX_SYMBOL_VALUE :
	678	* Maximum symbol value authorized.
	679	* Required for proper stack allocation */
	680	#ifndef FSE_MAX_SYMBOL_VALUE
	681	# define FSE_MAX_SYMBOL_VALUE 255
	682	#endif
	683
	684	/* **************************************************************
	685	* template functions type & suffix
	686	****************************************************************/
	687	#define FSE_FUNCTION_TYPE BYTE
	688	#define FSE_FUNCTION_EXTENSION
	689	#define FSE_DECODE_TYPE FSE_decode_t
	690
	691
	692	#endif /* !FSE_COMMONDEFS_ONLY */
	693
	694
	695	/* ***************************************************************
	696	* Constants
	697	*****************************************************************/
	698	#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2)
	699	#define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG)
	700	#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1)
	701	#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2)
	702	#define FSE_MIN_TABLELOG 5
	703
	704	#define FSE_TABLELOG_ABSOLUTE_MAX 15
	705	#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
	706	# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
	707	#endif
	708
	709	#define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3)
	710
	711
	712	#endif /* FSE_STATIC_LINKING_ONLY */
	713
	714
	715	#if defined (__cplusplus)
	716	}
	717	#endif

stage1/zstd/lib/common/fse_decompress.c created+403

...	...	@@ -0,0 +1,403 @@
	1	/* ******************************************************************
	2	* FSE : Finite State Entropy decoder
	3	* Copyright (c) Yann Collet, Facebook, Inc.
	4	*
	5	* You can contact the author at :
	6	* - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
	7	* - Public forum : https://groups.google.com/forum/#!forum/lz4c
	8	*
	9	* This source code is licensed under both the BSD-style license (found in the
	10	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	11	* in the COPYING file in the root directory of this source tree).
	12	* You may select, at your option, one of the above-listed licenses.
	13	****************************************************************** */
	14
	15
	16	/* **************************************************************
	17	* Includes
	18	****************************************************************/
	19	#include "debug.h" /* assert */
	20	#include "bitstream.h"
	21	#include "compiler.h"
	22	#define FSE_STATIC_LINKING_ONLY
	23	#include "fse.h"
	24	#include "error_private.h"
	25	#define ZSTD_DEPS_NEED_MALLOC
	26	#include "zstd_deps.h"
	27
	28
	29	/* **************************************************************
	30	* Error Management
	31	****************************************************************/
	32	#define FSE_isError ERR_isError
	33	#define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only after variable declarations */
	34
	35
	36	/* **************************************************************
	37	* Templates
	38	****************************************************************/
	39	/*
	40	designed to be included
	41	for type-specific functions (template emulation in C)
	42	Objective is to write these functions only once, for improved maintenance
	43	*/
	44
	45	/* safety checks */
	46	#ifndef FSE_FUNCTION_EXTENSION
	47	# error "FSE_FUNCTION_EXTENSION must be defined"
	48	#endif
	49	#ifndef FSE_FUNCTION_TYPE
	50	# error "FSE_FUNCTION_TYPE must be defined"
	51	#endif
	52
	53	/* Function names */
	54	#define FSE_CAT(X,Y) X##Y
	55	#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
	56	#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
	57
	58
	59	/* Function templates */
	60	FSE_DTable* FSE_createDTable (unsigned tableLog)
	61	{
	62	if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
	63	return (FSE_DTable)ZSTD_malloc( FSE_DTABLE_SIZE_U32(tableLog) sizeof (U32) );
	64	}
	65
	66	void FSE_freeDTable (FSE_DTable* dt)
	67	{
	68	ZSTD_free(dt);
	69	}
	70
	71	static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
	72	{
	73	void* const tdPtr = dt+1; /* because dt is unsigned, 32-bits aligned on 32-bits /
	74	FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr);
	75	U16* symbolNext = (U16*)workSpace;
	76	BYTE* spread = (BYTE*)(symbolNext + maxSymbolValue + 1);
	77
	78	U32 const maxSV1 = maxSymbolValue + 1;
	79	U32 const tableSize = 1 << tableLog;
	80	U32 highThreshold = tableSize-1;
	81
	82	/* Sanity Checks */
	83	if (FSE_BUILD_DTABLE_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(maxSymbolValue_tooLarge);
	84	if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge);
	85	if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
	86
	87	/* Init, lay down lowprob symbols */
	88	{ FSE_DTableHeader DTableH;
	89	DTableH.tableLog = (U16)tableLog;
	90	DTableH.fastMode = 1;
	91	{ S16 const largeLimit= (S16)(1 << (tableLog-1));
	92	U32 s;
	93	for (s=0; s<maxSV1; s++) {
	94	if (normalizedCounter[s]==-1) {
	95	tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
	96	symbolNext[s] = 1;
	97	} else {
	98	if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
	99	symbolNext[s] = normalizedCounter[s];
	100	} } }
	101	ZSTD_memcpy(dt, &DTableH, sizeof(DTableH));
	102	}
	103
	104	/* Spread symbols */
	105	if (highThreshold == tableSize - 1) {
	106	size_t const tableMask = tableSize-1;
	107	size_t const step = FSE_TABLESTEP(tableSize);
	108	/* First lay down the symbols in order.
	109	* We use a uint64_t to lay down 8 bytes at a time. This reduces branch
	110	* misses since small blocks generally have small table logs, so nearly
	111	* all symbols have counts <= 8. We ensure we have 8 bytes at the end of
	112	* our buffer to handle the over-write.
	113	*/
	114	{
	115	U64 const add = 0x0101010101010101ull;
	116	size_t pos = 0;
	117	U64 sv = 0;
	118	U32 s;
	119	for (s=0; s<maxSV1; ++s, sv += add) {
	120	int i;
	121	int const n = normalizedCounter[s];
	122	MEM_write64(spread + pos, sv);
	123	for (i = 8; i < n; i += 8) {
	124	MEM_write64(spread + pos + i, sv);
	125	}
	126	pos += n;
	127	}
	128	}
	129	/* Now we spread those positions across the table.
	130	* The benefit of doing it in two stages is that we avoid the the
	131	* variable size inner loop, which caused lots of branch misses.
	132	* Now we can run through all the positions without any branch misses.
	133	* We unroll the loop twice, since that is what emperically worked best.
	134	*/
	135	{
	136	size_t position = 0;
	137	size_t s;
	138	size_t const unroll = 2;
	139	assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */
	140	for (s = 0; s < (size_t)tableSize; s += unroll) {
	141	size_t u;
	142	for (u = 0; u < unroll; ++u) {
	143	size_t const uPosition = (position + (u * step)) & tableMask;
	144	tableDecode[uPosition].symbol = spread[s + u];
	145	}
	146	position = (position + (unroll * step)) & tableMask;
	147	}
	148	assert(position == 0);
	149	}
	150	} else {
	151	U32 const tableMask = tableSize-1;
	152	U32 const step = FSE_TABLESTEP(tableSize);
	153	U32 s, position = 0;
	154	for (s=0; s<maxSV1; s++) {
	155	int i;
	156	for (i=0; i<normalizedCounter[s]; i++) {
	157	tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
	158	position = (position + step) & tableMask;
	159	while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */
	160	} }
	161	if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
	162	}
	163
	164	/* Build Decoding table */
	165	{ U32 u;
	166	for (u=0; u<tableSize; u++) {
	167	FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
	168	U32 const nextState = symbolNext[symbol]++;
	169	tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32(nextState) );
	170	tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
	171	} }
	172
	173	return 0;
	174	}
	175
	176	size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
	177	{
	178	return FSE_buildDTable_internal(dt, normalizedCounter, maxSymbolValue, tableLog, workSpace, wkspSize);
	179	}
	180
	181
	182	#ifndef FSE_COMMONDEFS_ONLY
	183
	184	/-******************************************************
	185	* Decompression (Byte symbols)
	186	*********************************************************/
	187	size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
	188	{
	189	void* ptr = dt;
	190	FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
	191	void* dPtr = dt + 1;
	192	FSE_decode_t* const cell = (FSE_decode_t*)dPtr;
	193
	194	DTableH->tableLog = 0;
	195	DTableH->fastMode = 0;
	196
	197	cell->newState = 0;
	198	cell->symbol = symbolValue;
	199	cell->nbBits = 0;
	200
	201	return 0;
	202	}
	203
	204
	205	size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
	206	{
	207	void* ptr = dt;
	208	FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
	209	void* dPtr = dt + 1;
	210	FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr;
	211	const unsigned tableSize = 1 << nbBits;
	212	const unsigned tableMask = tableSize - 1;
	213	const unsigned maxSV1 = tableMask+1;
	214	unsigned s;
	215
	216	/* Sanity checks */
	217	if (nbBits < 1) return ERROR(GENERIC); /* min size */
	218
	219	/* Build Decoding Table */
	220	DTableH->tableLog = (U16)nbBits;
	221	DTableH->fastMode = 1;
	222	for (s=0; s<maxSV1; s++) {
	223	dinfo[s].newState = 0;
	224	dinfo[s].symbol = (BYTE)s;
	225	dinfo[s].nbBits = (BYTE)nbBits;
	226	}
	227
	228	return 0;
	229	}
	230
	231	FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic(
	232	void* dst, size_t maxDstSize,
	233	const void* cSrc, size_t cSrcSize,
	234	const FSE_DTable* dt, const unsigned fast)
	235	{
	236	BYTE* const ostart = (BYTE*) dst;
	237	BYTE* op = ostart;
	238	BYTE* const omax = op + maxDstSize;
	239	BYTE* const olimit = omax-3;
	240
	241	BIT_DStream_t bitD;
	242	FSE_DState_t state1;
	243	FSE_DState_t state2;
	244
	245	/* Init */
	246	CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize));
	247
	248	FSE_initDState(&state1, &bitD, dt);
	249	FSE_initDState(&state2, &bitD, dt);
	250
	251	#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
	252
	253	/* 4 symbols per loop */
	254	for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) & (op<olimit) ; op+=4) {
	255	op[0] = FSE_GETSYMBOL(&state1);
	256
	257	if (FSE_MAX_TABLELOG2+7 > sizeof(bitD.bitContainer)8) /* This test must be static */
	258	BIT_reloadDStream(&bitD);
	259
	260	op[1] = FSE_GETSYMBOL(&state2);
	261
	262	if (FSE_MAX_TABLELOG4+7 > sizeof(bitD.bitContainer)8) /* This test must be static */
	263	{ if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } }
	264
	265	op[2] = FSE_GETSYMBOL(&state1);
	266
	267	if (FSE_MAX_TABLELOG2+7 > sizeof(bitD.bitContainer)8) /* This test must be static */
	268	BIT_reloadDStream(&bitD);
	269
	270	op[3] = FSE_GETSYMBOL(&state2);
	271	}
	272
	273	/* tail */
	274	/* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */
	275	while (1) {
	276	if (op>(omax-2)) return ERROR(dstSize_tooSmall);
	277	*op++ = FSE_GETSYMBOL(&state1);
	278	if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
	279	*op++ = FSE_GETSYMBOL(&state2);
	280	break;
	281	}
	282
	283	if (op>(omax-2)) return ERROR(dstSize_tooSmall);
	284	*op++ = FSE_GETSYMBOL(&state2);
	285	if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
	286	*op++ = FSE_GETSYMBOL(&state1);
	287	break;
	288	} }
	289
	290	return op-ostart;
	291	}
	292
	293
	294	size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
	295	const void* cSrc, size_t cSrcSize,
	296	const FSE_DTable* dt)
	297	{
	298	const void* ptr = dt;
	299	const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
	300	const U32 fastMode = DTableH->fastMode;
	301
	302	/* select fast mode (static) */
	303	if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
	304	return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
	305	}
	306
	307
	308	size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
	309	{
	310	return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, /* bmi2 */ 0);
	311	}
	312
	313	typedef struct {
	314	short ncount[FSE_MAX_SYMBOL_VALUE + 1];
	315	FSE_DTable dtable[1]; /* Dynamically sized */
	316	} FSE_DecompressWksp;
	317
	318
	319	FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body(
	320	void* dst, size_t dstCapacity,
	321	const void* cSrc, size_t cSrcSize,
	322	unsigned maxLog, void* workSpace, size_t wkspSize,
	323	int bmi2)
	324	{
	325	const BYTE* const istart = (const BYTE*)cSrc;
	326	const BYTE* ip = istart;
	327	unsigned tableLog;
	328	unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
	329	FSE_DecompressWksp* const wksp = (FSE_DecompressWksp*)workSpace;
	330
	331	DEBUG_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0);
	332	if (wkspSize < sizeof(*wksp)) return ERROR(GENERIC);
	333
	334	/* normal FSE decoding mode */
	335	{
	336	size_t const NCountLength = FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2);
	337	if (FSE_isError(NCountLength)) return NCountLength;
	338	if (tableLog > maxLog) return ERROR(tableLog_tooLarge);
	339	assert(NCountLength <= cSrcSize);
	340	ip += NCountLength;
	341	cSrcSize -= NCountLength;
	342	}
	343
	344	if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge);
	345	workSpace = wksp->dtable + FSE_DTABLE_SIZE_U32(tableLog);
	346	wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog);
	347
	348	CHECK_F( FSE_buildDTable_internal(wksp->dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) );
	349
	350	{
	351	const void* ptr = wksp->dtable;
	352	const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
	353	const U32 fastMode = DTableH->fastMode;
	354
	355	/* select fast mode (static) */
	356	if (fastMode) return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 1);
	357	return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 0);
	358	}
	359	}
	360
	361	/* Avoids the FORCE_INLINE of the _body() function. */
	362	static size_t FSE_decompress_wksp_body_default(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
	363	{
	364	return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 0);
	365	}
	366
	367	#if DYNAMIC_BMI2
	368	BMI2_TARGET_ATTRIBUTE static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
	369	{
	370	return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1);
	371	}
	372	#endif
	373
	374	size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2)
	375	{
	376	#if DYNAMIC_BMI2
	377	if (bmi2) {
	378	return FSE_decompress_wksp_body_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
	379	}
	380	#endif
	381	(void)bmi2;
	382	return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
	383	}
	384
	385
	386	typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
	387
	388	#ifndef ZSTD_NO_UNUSED_FUNCTIONS
	389	size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) {
	390	U32 wksp[FSE_BUILD_DTABLE_WKSP_SIZE_U32(FSE_TABLELOG_ABSOLUTE_MAX, FSE_MAX_SYMBOL_VALUE)];
	391	return FSE_buildDTable_wksp(dt, normalizedCounter, maxSymbolValue, tableLog, wksp, sizeof(wksp));
	392	}
	393
	394	size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize)
	395	{
	396	/* Static analyzer seems unable to understand this table will be properly initialized later */
	397	U32 wksp[FSE_DECOMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
	398	return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, FSE_MAX_TABLELOG, wksp, sizeof(wksp));
	399	}
	400	#endif
	401
	402
	403	#endif /* FSE_COMMONDEFS_ONLY */

stage1/zstd/lib/common/huf.h created+364

...	...	@@ -0,0 +1,364 @@
	1	/* ******************************************************************
	2	* huff0 huffman codec,
	3	* part of Finite State Entropy library
	4	* Copyright (c) Yann Collet, Facebook, Inc.
	5	*
	6	* You can contact the author at :
	7	* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
	8	*
	9	* This source code is licensed under both the BSD-style license (found in the
	10	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	11	* in the COPYING file in the root directory of this source tree).
	12	* You may select, at your option, one of the above-listed licenses.
	13	****************************************************************** */
	14
	15	#if defined (__cplusplus)
	16	extern "C" {
	17	#endif
	18
	19	#ifndef HUF_H_298734234
	20	#define HUF_H_298734234
	21
	22	/* * Dependencies * */
	23	#include "zstd_deps.h" /* size_t */
	24
	25
	26	/* * library symbols visibility * */
	27	/* Note : when linking with -fvisibility=hidden on gcc, or by default on Visual,
	28	* HUF symbols remain "private" (internal symbols for library only).
	29	* Set macro FSE_DLL_EXPORT to 1 if you want HUF symbols visible on DLL interface */
	30	#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
	31	# define HUF_PUBLIC_API __attribute__ ((visibility ("default")))
	32	#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */
	33	# define HUF_PUBLIC_API __declspec(dllexport)
	34	#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
	35	# define HUF_PUBLIC_API __declspec(dllimport) /* not required, just to generate faster code (saves a function pointer load from IAT and an indirect jump) */
	36	#else
	37	# define HUF_PUBLIC_API
	38	#endif
	39
	40
	41	/* ========================== */
	42	/* * simple functions * */
	43	/* ========================== */
	44
	45	/** HUF_compress() :
	46	* Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
	47	* 'dst' buffer must be already allocated.
	48	* Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
	49	* `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.
	50	* @return : size of compressed data (<= `dstCapacity`).
	51	* Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
	52	* if HUF_isError(return), compression failed (more details using HUF_getErrorName())
	53	*/
	54	HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity,
	55	const void* src, size_t srcSize);
	56
	57	/** HUF_decompress() :
	58	* Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
	59	* into already allocated buffer 'dst', of minimum size 'dstSize'.
	60	* `originalSize` : must be the *exact* size of original (uncompressed) data.
	61	* Note : in contrast with FSE, HUF_decompress can regenerate
	62	* RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
	63	* because it knows size to regenerate (originalSize).
	64	* @return : size of regenerated data (== originalSize),
	65	* or an error code, which can be tested using HUF_isError()
	66	*/
	67	HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize,
	68	const void* cSrc, size_t cSrcSize);
	69
	70
	71	/* * Tool functions * */
	72	#define HUF_BLOCKSIZE_MAX (128 * 1024) /*< maximum input size for a single block compressed with HUF_compress /
	73	HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /*< maximum compressed size (worst case) /
	74
	75	/* Error Management */
	76	HUF_PUBLIC_API unsigned HUF_isError(size_t code); /*< tells if a return value is an error code /
	77	HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /*< provides error code string (useful for debugging) /
	78
	79
	80	/* * Advanced function * */
	81
	82	/** HUF_compress2() :
	83	* Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`.
	84	* `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX .
	85	* `tableLog` must be `<= HUF_TABLELOG_MAX` . */
	86	HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity,
	87	const void* src, size_t srcSize,
	88	unsigned maxSymbolValue, unsigned tableLog);
	89
	90	/** HUF_compress4X_wksp() :
	91	* Same as HUF_compress2(), but uses externally allocated `workSpace`.
	92	* `workspace` must be at least as large as HUF_WORKSPACE_SIZE */
	93	#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */)
	94	#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64))
	95	HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
	96	const void* src, size_t srcSize,
	97	unsigned maxSymbolValue, unsigned tableLog,
	98	void* workSpace, size_t wkspSize);
	99
	100	#endif /* HUF_H_298734234 */
	101
	102	/* ******************************************************************
	103	* WARNING !!
	104	* The following section contains advanced and experimental definitions
	105	* which shall never be used in the context of a dynamic library,
	106	* because they are not guaranteed to remain stable in the future.
	107	* Only consider them in association with static linking.
	108	* *****************************************************************/
	109	#if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY)
	110	#define HUF_H_HUF_STATIC_LINKING_ONLY
	111
	112	/* * Dependencies * */
	113	#include "mem.h" /* U32 */
	114	#define FSE_STATIC_LINKING_ONLY
	115	#include "fse.h"
	116
	117
	118	/* * Constants * */
	119	#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */
	120	#define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */
	121	#define HUF_SYMBOLVALUE_MAX 255
	122
	123	#define HUF_TABLELOG_ABSOLUTEMAX 12 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
	124	#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
	125	# error "HUF_TABLELOG_MAX is too large !"
	126	#endif
	127
	128
	129	/* ****************************************
	130	* Static allocation
	131	******************************************/
	132	/* HUF buffer bounds */
	133	#define HUF_CTABLEBOUND 129
	134	#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true when incompressible is pre-filtered with fast heuristic */
	135	#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
	136
	137	/* static allocation of HUF's Compression Table */
	138	/* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */
	139	typedef size_t HUF_CElt; /* consider it an incomplete type */
	140	#define HUF_CTABLE_SIZE_ST(maxSymbolValue) ((maxSymbolValue)+2) /* Use tables of size_t, for proper alignment */
	141	#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_ST(maxSymbolValue) * sizeof(size_t))
	142	#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
	143	HUF_CElt name[HUF_CTABLE_SIZE_ST(maxSymbolValue)] /* no final ; */
	144
	145	/* static allocation of HUF's DTable */
	146	typedef U32 HUF_DTable;
	147	#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))
	148	#define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \
	149	HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) }
	150	#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
	151	HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }
	152
	153
	154	/* ****************************************
	155	* Advanced decompression functions
	156	******************************************/
	157	size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< single-symbol decoder /
	158	#ifndef HUF_FORCE_DECOMPRESS_X1
	159	size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< double-symbols decoder /
	160	#endif
	161
	162	size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< decodes RLE and uncompressed /
	163	size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< considers RLE and uncompressed as errors /
	164	size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /*< considers RLE and uncompressed as errors /
	165	size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< single-symbol decoder /
	166	size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /*< single-symbol decoder /
	167	#ifndef HUF_FORCE_DECOMPRESS_X1
	168	size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< double-symbols decoder /
	169	size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /*< double-symbols decoder /
	170	#endif
	171
	172
	173	/* ****************************************
	174	* HUF detailed API
	175	* ****************************************/
	176
	177	/*! HUF_compress() does the following:
	178	* 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h")
	179	* 2. (optional) refine tableLog using HUF_optimalTableLog()
	180	* 3. build Huffman table from count using HUF_buildCTable()
	181	* 4. save Huffman table to memory buffer using HUF_writeCTable()
	182	* 5. encode the data stream using HUF_compress4X_usingCTable()
	183	*
	184	* The following API allows targeting specific sub-functions for advanced tasks.
	185	* For example, it's possible to compress several blocks using the same 'CTable',
	186	* or to save and regenerate 'CTable' using external methods.
	187	*/
	188	unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
	189	size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */
	190	size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
	191	size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize);
	192	size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
	193	size_t HUF_compress4X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2);
	194	size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
	195	int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
	196
	197	typedef enum {
	198	HUF_repeat_none, /*< Cannot use the previous table /
	199	HUF_repeat_check, /*< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat /
	200	HUF_repeat_valid /*< Can use the previous table and it is assumed to be valid /
	201	} HUF_repeat;
	202	/** HUF_compress4X_repeat() :
	203	* Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
	204	* If it uses hufTable it does not modify hufTable or repeat.
	205	* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
	206	* If preferRepeat then the old table will always be used if valid.
	207	* If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
	208	size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
	209	const void* src, size_t srcSize,
	210	unsigned maxSymbolValue, unsigned tableLog,
	211	void* workSpace, size_t wkspSize, /*< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE /
	212	HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible);
	213
	214	/** HUF_buildCTable_wksp() :
	215	* Same as HUF_buildCTable(), but using externally allocated scratch buffer.
	216	* `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE.
	217	*/
	218	#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1)
	219	#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
	220	size_t HUF_buildCTable_wksp (HUF_CElt* tree,
	221	const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
	222	void* workSpace, size_t wkspSize);
	223
	224	/*! HUF_readStats() :
	225	* Read compact Huffman tree, saved by HUF_writeCTable().
	226	* `huffWeight` is destination buffer.
	227	* @return : size read from `src` , or an error Code .
	228	* Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
	229	size_t HUF_readStats(BYTE* huffWeight, size_t hwSize,
	230	U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
	231	const void* src, size_t srcSize);
	232
	233	/*! HUF_readStats_wksp() :
	234	* Same as HUF_readStats() but takes an external workspace which must be
	235	* 4-byte aligned and its size must be >= HUF_READ_STATS_WORKSPACE_SIZE.
	236	* If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
	237	*/
	238	#define HUF_READ_STATS_WORKSPACE_SIZE_U32 FSE_DECOMPRESS_WKSP_SIZE_U32(6, HUF_TABLELOG_MAX-1)
	239	#define HUF_READ_STATS_WORKSPACE_SIZE (HUF_READ_STATS_WORKSPACE_SIZE_U32 * sizeof(unsigned))
	240	size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize,
	241	U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
	242	const void* src, size_t srcSize,
	243	void* workspace, size_t wkspSize,
	244	int bmi2);
	245
	246	/** HUF_readCTable() :
	247	* Loading a CTable saved with HUF_writeCTable() */
	248	size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights);
	249
	250	/** HUF_getNbBitsFromCTable() :
	251	* Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX
	252	* Note 1 : is not inlined, as HUF_CElt definition is private */
	253	U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue);
	254
	255	/*
	256	* HUF_decompress() does the following:
	257	* 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics
	258	* 2. build Huffman table from save, using HUF_readDTableX?()
	259	* 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable()
	260	*/
	261
	262	/** HUF_selectDecoder() :
	263	* Tells which decoder is likely to decode faster,
	264	* based on a set of pre-computed metrics.
	265	* @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 .
	266	* Assumption : 0 < dstSize <= 128 KB */
	267	U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
	268
	269	/**
	270	* The minimum workspace size for the `workSpace` used in
	271	* HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp().
	272	*
	273	* The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when
	274	* HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15.
	275	* Buffer overflow errors may potentially occur if code modifications result in
	276	* a required workspace size greater than that specified in the following
	277	* macro.
	278	*/
	279	#define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9))
	280	#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
	281
	282	#ifndef HUF_FORCE_DECOMPRESS_X2
	283	size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize);
	284	size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
	285	#endif
	286	#ifndef HUF_FORCE_DECOMPRESS_X1
	287	size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
	288	size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
	289	#endif
	290
	291	size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
	292	#ifndef HUF_FORCE_DECOMPRESS_X2
	293	size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
	294	#endif
	295	#ifndef HUF_FORCE_DECOMPRESS_X1
	296	size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
	297	#endif
	298
	299
	300	/* ====================== */
	301	/* single stream variants */
	302	/* ====================== */
	303
	304	size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
	305	size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /*< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U64 U64 /
	306	size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
	307	size_t HUF_compress1X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2);
	308	/** HUF_compress1X_repeat() :
	309	* Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
	310	* If it uses hufTable it does not modify hufTable or repeat.
	311	* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
	312	* If preferRepeat then the old table will always be used if valid.
	313	* If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
	314	size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
	315	const void* src, size_t srcSize,
	316	unsigned maxSymbolValue, unsigned tableLog,
	317	void* workSpace, size_t wkspSize, /*< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE /
	318	HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible);
	319
	320	size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
	321	#ifndef HUF_FORCE_DECOMPRESS_X1
	322	size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
	323	#endif
	324
	325	size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
	326	size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);
	327	#ifndef HUF_FORCE_DECOMPRESS_X2
	328	size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< single-symbol decoder /
	329	size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /*< single-symbol decoder /
	330	#endif
	331	#ifndef HUF_FORCE_DECOMPRESS_X1
	332	size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< double-symbols decoder /
	333	size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /*< double-symbols decoder /
	334	#endif
	335
	336	size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /*< automatic selection of sing or double symbol decoder, based on DTable /
	337	#ifndef HUF_FORCE_DECOMPRESS_X2
	338	size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
	339	#endif
	340	#ifndef HUF_FORCE_DECOMPRESS_X1
	341	size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
	342	#endif
	343
	344	/* BMI2 variants.
	345	* If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
	346	*/
	347	size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
	348	#ifndef HUF_FORCE_DECOMPRESS_X2
	349	size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
	350	#endif
	351	size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
	352	size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
	353	#ifndef HUF_FORCE_DECOMPRESS_X2
	354	size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2);
	355	#endif
	356	#ifndef HUF_FORCE_DECOMPRESS_X1
	357	size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2);
	358	#endif
	359
	360	#endif /* HUF_STATIC_LINKING_ONLY */
	361
	362	#if defined (__cplusplus)
	363	}
	364	#endif

stage1/zstd/lib/common/mem.h created+442

...	...	@@ -0,0 +1,442 @@
	1	/*
	2	* Copyright (c) Yann Collet, Facebook, Inc.
	3	* All rights reserved.
	4	*
	5	* This source code is licensed under both the BSD-style license (found in the
	6	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	7	* in the COPYING file in the root directory of this source tree).
	8	* You may select, at your option, one of the above-listed licenses.
	9	*/
	10
	11	#ifndef MEM_H_MODULE
	12	#define MEM_H_MODULE
	13
	14	#if defined (__cplusplus)
	15	extern "C" {
	16	#endif
	17
	18	/-***************************************
	19	* Dependencies
	20	******************************************/
	21	#include <stddef.h> /* size_t, ptrdiff_t */
	22	#include "compiler.h" /* __has_builtin */
	23	#include "debug.h" /* DEBUG_STATIC_ASSERT */
	24	#include "zstd_deps.h" /* ZSTD_memcpy */
	25
	26
	27	/-***************************************
	28	* Compiler specifics
	29	******************************************/
	30	#if defined(_MSC_VER) /* Visual Studio */
	31	# include <stdlib.h> /* _byteswap_ulong */
	32	# include <intrin.h> /* _byteswap_* */
	33	#endif
	34	#if defined(__GNUC__)
	35	# define MEM_STATIC static __inline __attribute__((unused))
	36	#elif defined (__cplusplus) \|\| (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
	37	# define MEM_STATIC static inline
	38	#elif defined(_MSC_VER)
	39	# define MEM_STATIC static __inline
	40	#else
	41	# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
	42	#endif
	43
	44	/-*************************************************************
	45	* Basic Types
	46	*****************************************************************/
	47	#if !defined (__VMS) && (defined (__cplusplus) \|\| (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
	48	# if defined(_AIX)
	49	# include <inttypes.h>
	50	# else
	51	# include <stdint.h> /* intptr_t */
	52	# endif
	53	typedef uint8_t BYTE;
	54	typedef uint8_t U8;
	55	typedef int8_t S8;
	56	typedef uint16_t U16;
	57	typedef int16_t S16;
	58	typedef uint32_t U32;
	59	typedef int32_t S32;
	60	typedef uint64_t U64;
	61	typedef int64_t S64;
	62	#else
	63	# include <limits.h>
	64	#if CHAR_BIT != 8
	65	# error "this implementation requires char to be exactly 8-bit type"
	66	#endif
	67	typedef unsigned char BYTE;
	68	typedef unsigned char U8;
	69	typedef signed char S8;
	70	#if USHRT_MAX != 65535
	71	# error "this implementation requires short to be exactly 16-bit type"
	72	#endif
	73	typedef unsigned short U16;
	74	typedef signed short S16;
	75	#if UINT_MAX != 4294967295
	76	# error "this implementation requires int to be exactly 32-bit type"
	77	#endif
	78	typedef unsigned int U32;
	79	typedef signed int S32;
	80	/* note : there are no limits defined for long long type in C90.
	81	* limits exist in C99, however, in such case, <stdint.h> is preferred */
	82	typedef unsigned long long U64;
	83	typedef signed long long S64;
	84	#endif
	85
	86
	87	/-*************************************************************
	88	* Memory I/O API
	89	*****************************************************************/
	90	/=== Static platform detection ===/
	91	MEM_STATIC unsigned MEM_32bits(void);
	92	MEM_STATIC unsigned MEM_64bits(void);
	93	MEM_STATIC unsigned MEM_isLittleEndian(void);
	94
	95	/=== Native unaligned read/write ===/
	96	MEM_STATIC U16 MEM_read16(const void* memPtr);
	97	MEM_STATIC U32 MEM_read32(const void* memPtr);
	98	MEM_STATIC U64 MEM_read64(const void* memPtr);
	99	MEM_STATIC size_t MEM_readST(const void* memPtr);
	100
	101	MEM_STATIC void MEM_write16(void* memPtr, U16 value);
	102	MEM_STATIC void MEM_write32(void* memPtr, U32 value);
	103	MEM_STATIC void MEM_write64(void* memPtr, U64 value);
	104
	105	/=== Little endian unaligned read/write ===/
	106	MEM_STATIC U16 MEM_readLE16(const void* memPtr);
	107	MEM_STATIC U32 MEM_readLE24(const void* memPtr);
	108	MEM_STATIC U32 MEM_readLE32(const void* memPtr);
	109	MEM_STATIC U64 MEM_readLE64(const void* memPtr);
	110	MEM_STATIC size_t MEM_readLEST(const void* memPtr);
	111
	112	MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val);
	113	MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val);
	114	MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32);
	115	MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64);
	116	MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val);
	117
	118	/=== Big endian unaligned read/write ===/
	119	MEM_STATIC U32 MEM_readBE32(const void* memPtr);
	120	MEM_STATIC U64 MEM_readBE64(const void* memPtr);
	121	MEM_STATIC size_t MEM_readBEST(const void* memPtr);
	122
	123	MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32);
	124	MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64);
	125	MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val);
	126
	127	/=== Byteswap ===/
	128	MEM_STATIC U32 MEM_swap32(U32 in);
	129	MEM_STATIC U64 MEM_swap64(U64 in);
	130	MEM_STATIC size_t MEM_swapST(size_t in);
	131
	132
	133	/-*************************************************************
	134	* Memory I/O Implementation
	135	*****************************************************************/
	136	/* MEM_FORCE_MEMORY_ACCESS :
	137	* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
	138	* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
	139	* The below switch allow to select different access method for improved performance.
	140	* Method 0 (default) : use `memcpy()`. Safe and portable.
	141	* Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable).
	142	* This method is safe if your compiler supports it, and generally as fast or faster than `memcpy`.
	143	* Method 2 : direct access. This method is portable but violate C standard.
	144	* It can generate buggy code on targets depending on alignment.
	145	* In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6)
	146	* See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
	147	* Prefer these methods in priority order (0 > 1 > 2)
	148	*/
	149	#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
	150	# if defined(__INTEL_COMPILER) \|\| defined(__GNUC__) \|\| defined(__ICCARM__)
	151	# define MEM_FORCE_MEMORY_ACCESS 1
	152	# endif
	153	#endif
	154
	155	MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; }
	156	MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; }
	157
	158	MEM_STATIC unsigned MEM_isLittleEndian(void)
	159	{
	160	#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
	161	return 1;
	162	#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
	163	return 0;
	164	#elif defined(__clang__) && __LITTLE_ENDIAN__
	165	return 1;
	166	#elif defined(__clang__) && __BIG_ENDIAN__
	167	return 0;
	168	#elif defined(_MSC_VER) && (_M_AMD64 \|\| _M_IX86)
	169	return 1;
	170	#elif defined(__DMC__) && defined(_M_IX86)
	171	return 1;
	172	#else
	173	const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
	174	return one.c[0];
	175	#endif
	176	}
	177
	178	#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
	179
	180	/* violates C standard, by lying on structure alignment.
	181	Only use if no other choice to achieve best performance on target platform */
	182	MEM_STATIC U16 MEM_read16(const void* memPtr) { return (const U16) memPtr; }
	183	MEM_STATIC U32 MEM_read32(const void* memPtr) { return (const U32) memPtr; }
	184	MEM_STATIC U64 MEM_read64(const void* memPtr) { return (const U64) memPtr; }
	185	MEM_STATIC size_t MEM_readST(const void* memPtr) { return (const size_t) memPtr; }
	186
	187	MEM_STATIC void MEM_write16(void* memPtr, U16 value) { (U16)memPtr = value; }
	188	MEM_STATIC void MEM_write32(void* memPtr, U32 value) { (U32)memPtr = value; }
	189	MEM_STATIC void MEM_write64(void* memPtr, U64 value) { (U64)memPtr = value; }
	190
	191	#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
	192
	193	/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
	194	/* currently only defined for gcc and icc */
	195	#if defined(_MSC_VER) \|\| (defined(__INTEL_COMPILER) && defined(WIN32))
	196	__pragma( pack(push, 1) )
	197	typedef struct { U16 v; } unalign16;
	198	typedef struct { U32 v; } unalign32;
	199	typedef struct { U64 v; } unalign64;
	200	typedef struct { size_t v; } unalignArch;
	201	__pragma( pack(pop) )
	202	#else
	203	typedef struct { U16 v; } __attribute__((packed)) unalign16;
	204	typedef struct { U32 v; } __attribute__((packed)) unalign32;
	205	typedef struct { U64 v; } __attribute__((packed)) unalign64;
	206	typedef struct { size_t v; } __attribute__((packed)) unalignArch;
	207	#endif
	208
	209	MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; }
	210	MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; }
	211	MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; }
	212	MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; }
	213
	214	MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; }
	215	MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; }
	216	MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; }
	217
	218	#else
	219
	220	/* default method, safe and standard.
	221	can sometimes prove slower */
	222
	223	MEM_STATIC U16 MEM_read16(const void* memPtr)
	224	{
	225	U16 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
	226	}
	227
	228	MEM_STATIC U32 MEM_read32(const void* memPtr)
	229	{
	230	U32 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
	231	}
	232
	233	MEM_STATIC U64 MEM_read64(const void* memPtr)
	234	{
	235	U64 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
	236	}
	237
	238	MEM_STATIC size_t MEM_readST(const void* memPtr)
	239	{
	240	size_t val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val;
	241	}
	242
	243	MEM_STATIC void MEM_write16(void* memPtr, U16 value)
	244	{
	245	ZSTD_memcpy(memPtr, &value, sizeof(value));
	246	}
	247
	248	MEM_STATIC void MEM_write32(void* memPtr, U32 value)
	249	{
	250	ZSTD_memcpy(memPtr, &value, sizeof(value));
	251	}
	252
	253	MEM_STATIC void MEM_write64(void* memPtr, U64 value)
	254	{
	255	ZSTD_memcpy(memPtr, &value, sizeof(value));
	256	}
	257
	258	#endif /* MEM_FORCE_MEMORY_ACCESS */
	259
	260	MEM_STATIC U32 MEM_swap32(U32 in)
	261	{
	262	#if defined(_MSC_VER) /* Visual Studio */
	263	return _byteswap_ulong(in);
	264	#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
	265	\|\| (defined(__clang__) && __has_builtin(__builtin_bswap32))
	266	return __builtin_bswap32(in);
	267	#else
	268	return ((in << 24) & 0xff000000 ) \|
	269	((in << 8) & 0x00ff0000 ) \|
	270	((in >> 8) & 0x0000ff00 ) \|
	271	((in >> 24) & 0x000000ff );
	272	#endif
	273	}
	274
	275	MEM_STATIC U64 MEM_swap64(U64 in)
	276	{
	277	#if defined(_MSC_VER) /* Visual Studio */
	278	return _byteswap_uint64(in);
	279	#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
	280	\|\| (defined(__clang__) && __has_builtin(__builtin_bswap64))
	281	return __builtin_bswap64(in);
	282	#else
	283	return ((in << 56) & 0xff00000000000000ULL) \|
	284	((in << 40) & 0x00ff000000000000ULL) \|
	285	((in << 24) & 0x0000ff0000000000ULL) \|
	286	((in << 8) & 0x000000ff00000000ULL) \|
	287	((in >> 8) & 0x00000000ff000000ULL) \|
	288	((in >> 24) & 0x0000000000ff0000ULL) \|
	289	((in >> 40) & 0x000000000000ff00ULL) \|
	290	((in >> 56) & 0x00000000000000ffULL);
	291	#endif
	292	}
	293
	294	MEM_STATIC size_t MEM_swapST(size_t in)
	295	{
	296	if (MEM_32bits())
	297	return (size_t)MEM_swap32((U32)in);
	298	else
	299	return (size_t)MEM_swap64((U64)in);
	300	}
	301
	302	/=== Little endian r/w ===/
	303
	304	MEM_STATIC U16 MEM_readLE16(const void* memPtr)
	305	{
	306	if (MEM_isLittleEndian())
	307	return MEM_read16(memPtr);
	308	else {
	309	const BYTE* p = (const BYTE*)memPtr;
	310	return (U16)(p[0] + (p[1]<<8));
	311	}
	312	}
	313
	314	MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
	315	{
	316	if (MEM_isLittleEndian()) {
	317	MEM_write16(memPtr, val);
	318	} else {
	319	BYTE* p = (BYTE*)memPtr;
	320	p[0] = (BYTE)val;
	321	p[1] = (BYTE)(val>>8);
	322	}
	323	}
	324
	325	MEM_STATIC U32 MEM_readLE24(const void* memPtr)
	326	{
	327	return (U32)MEM_readLE16(memPtr) + ((U32)(((const BYTE*)memPtr)[2]) << 16);
	328	}
	329
	330	MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val)
	331	{
	332	MEM_writeLE16(memPtr, (U16)val);
	333	((BYTE*)memPtr)[2] = (BYTE)(val>>16);
	334	}
	335
	336	MEM_STATIC U32 MEM_readLE32(const void* memPtr)
	337	{
	338	if (MEM_isLittleEndian())
	339	return MEM_read32(memPtr);
	340	else
	341	return MEM_swap32(MEM_read32(memPtr));
	342	}
	343
	344	MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
	345	{
	346	if (MEM_isLittleEndian())
	347	MEM_write32(memPtr, val32);
	348	else
	349	MEM_write32(memPtr, MEM_swap32(val32));
	350	}
	351
	352	MEM_STATIC U64 MEM_readLE64(const void* memPtr)
	353	{
	354	if (MEM_isLittleEndian())
	355	return MEM_read64(memPtr);
	356	else
	357	return MEM_swap64(MEM_read64(memPtr));
	358	}
	359
	360	MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
	361	{
	362	if (MEM_isLittleEndian())
	363	MEM_write64(memPtr, val64);
	364	else
	365	MEM_write64(memPtr, MEM_swap64(val64));
	366	}
	367
	368	MEM_STATIC size_t MEM_readLEST(const void* memPtr)
	369	{
	370	if (MEM_32bits())
	371	return (size_t)MEM_readLE32(memPtr);
	372	else
	373	return (size_t)MEM_readLE64(memPtr);
	374	}
	375
	376	MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
	377	{
	378	if (MEM_32bits())
	379	MEM_writeLE32(memPtr, (U32)val);
	380	else
	381	MEM_writeLE64(memPtr, (U64)val);
	382	}
	383
	384	/=== Big endian r/w ===/
	385
	386	MEM_STATIC U32 MEM_readBE32(const void* memPtr)
	387	{
	388	if (MEM_isLittleEndian())
	389	return MEM_swap32(MEM_read32(memPtr));
	390	else
	391	return MEM_read32(memPtr);
	392	}
	393
	394	MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32)
	395	{
	396	if (MEM_isLittleEndian())
	397	MEM_write32(memPtr, MEM_swap32(val32));
	398	else
	399	MEM_write32(memPtr, val32);
	400	}
	401
	402	MEM_STATIC U64 MEM_readBE64(const void* memPtr)
	403	{
	404	if (MEM_isLittleEndian())
	405	return MEM_swap64(MEM_read64(memPtr));
	406	else
	407	return MEM_read64(memPtr);
	408	}
	409
	410	MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64)
	411	{
	412	if (MEM_isLittleEndian())
	413	MEM_write64(memPtr, MEM_swap64(val64));
	414	else
	415	MEM_write64(memPtr, val64);
	416	}
	417
	418	MEM_STATIC size_t MEM_readBEST(const void* memPtr)
	419	{
	420	if (MEM_32bits())
	421	return (size_t)MEM_readBE32(memPtr);
	422	else
	423	return (size_t)MEM_readBE64(memPtr);
	424	}
	425
	426	MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
	427	{
	428	if (MEM_32bits())
	429	MEM_writeBE32(memPtr, (U32)val);
	430	else
	431	MEM_writeBE64(memPtr, (U64)val);
	432	}
	433
	434	/* code only tested on 32 and 64 bits systems */
	435	MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) \|\| (sizeof(size_t)==8)); }
	436
	437
	438	#if defined (__cplusplus)
	439	}
	440	#endif
	441
	442	#endif /* MEM_H_MODULE */

stage1/zstd/lib/common/pool.c created+355

...	...	@@ -0,0 +1,355 @@
	1	/*
	2	* Copyright (c) Yann Collet, Facebook, Inc.
	3	* All rights reserved.
	4	*
	5	* This source code is licensed under both the BSD-style license (found in the
	6	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	7	* in the COPYING file in the root directory of this source tree).
	8	* You may select, at your option, one of the above-listed licenses.
	9	*/
	10
	11
	12	/* ====== Dependencies ======= */
	13	#include "zstd_deps.h" /* size_t */
	14	#include "debug.h" /* assert */
	15	#include "zstd_internal.h" /* ZSTD_customMalloc, ZSTD_customFree */
	16	#include "pool.h"
	17
	18	/* ====== Compiler specifics ====== */
	19	#if defined(_MSC_VER)
	20	# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
	21	#endif
	22
	23
	24	#ifdef ZSTD_MULTITHREAD
	25
	26	#include "threading.h" /* pthread adaptation */
	27
	28	/* A job is a function and an opaque argument */
	29	typedef struct POOL_job_s {
	30	POOL_function function;
	31	void *opaque;
	32	} POOL_job;
	33
	34	struct POOL_ctx_s {
	35	ZSTD_customMem customMem;
	36	/* Keep track of the threads */
	37	ZSTD_pthread_t* threads;
	38	size_t threadCapacity;
	39	size_t threadLimit;
	40
	41	/* The queue is a circular buffer */
	42	POOL_job *queue;
	43	size_t queueHead;
	44	size_t queueTail;
	45	size_t queueSize;
	46
	47	/* The number of threads working on jobs */
	48	size_t numThreadsBusy;
	49	/* Indicates if the queue is empty */
	50	int queueEmpty;
	51
	52	/* The mutex protects the queue */
	53	ZSTD_pthread_mutex_t queueMutex;
	54	/* Condition variable for pushers to wait on when the queue is full */
	55	ZSTD_pthread_cond_t queuePushCond;
	56	/* Condition variables for poppers to wait on when the queue is empty */
	57	ZSTD_pthread_cond_t queuePopCond;
	58	/* Indicates if the queue is shutting down */
	59	int shutdown;
	60	};
	61
	62	/* POOL_thread() :
	63	* Work thread for the thread pool.
	64	* Waits for jobs and executes them.
	65	* @returns : NULL on failure else non-null.
	66	*/
	67	static void* POOL_thread(void* opaque) {
	68	POOL_ctx* const ctx = (POOL_ctx*)opaque;
	69	if (!ctx) { return NULL; }
	70	for (;;) {
	71	/* Lock the mutex and wait for a non-empty queue or until shutdown */
	72	ZSTD_pthread_mutex_lock(&ctx->queueMutex);
	73
	74	while ( ctx->queueEmpty
	75	\|\| (ctx->numThreadsBusy >= ctx->threadLimit) ) {
	76	if (ctx->shutdown) {
	77	/* even if !queueEmpty, (possible if numThreadsBusy >= threadLimit),
	78	* a few threads will be shutdown while !queueEmpty,
	79	* but enough threads will remain active to finish the queue */
	80	ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
	81	return opaque;
	82	}
	83	ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
	84	}
	85	/* Pop a job off the queue */
	86	{ POOL_job const job = ctx->queue[ctx->queueHead];
	87	ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize;
	88	ctx->numThreadsBusy++;
	89	ctx->queueEmpty = (ctx->queueHead == ctx->queueTail);
	90	/* Unlock the mutex, signal a pusher, and run the job */
	91	ZSTD_pthread_cond_signal(&ctx->queuePushCond);
	92	ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
	93
	94	job.function(job.opaque);
	95
	96	/* If the intended queue size was 0, signal after finishing job */
	97	ZSTD_pthread_mutex_lock(&ctx->queueMutex);
	98	ctx->numThreadsBusy--;
	99	if (ctx->queueSize == 1) {
	100	ZSTD_pthread_cond_signal(&ctx->queuePushCond);
	101	}
	102	ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
	103	}
	104	} /* for (;;) */
	105	assert(0); /* Unreachable */
	106	}
	107
	108	/* ZSTD_createThreadPool() : public access point */
	109	POOL_ctx* ZSTD_createThreadPool(size_t numThreads) {
	110	return POOL_create (numThreads, 0);
	111	}
	112
	113	POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
	114	return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
	115	}
	116
	117	POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
	118	ZSTD_customMem customMem)
	119	{
	120	POOL_ctx* ctx;
	121	/* Check parameters */
	122	if (!numThreads) { return NULL; }
	123	/* Allocate the context and zero initialize */
	124	ctx = (POOL_ctx*)ZSTD_customCalloc(sizeof(POOL_ctx), customMem);
	125	if (!ctx) { return NULL; }
	126	/* Initialize the job queue.
	127	* It needs one extra space since one space is wasted to differentiate
	128	* empty and full queues.
	129	*/
	130	ctx->queueSize = queueSize + 1;
	131	ctx->queue = (POOL_job)ZSTD_customMalloc(ctx->queueSize sizeof(POOL_job), customMem);
	132	ctx->queueHead = 0;
	133	ctx->queueTail = 0;
	134	ctx->numThreadsBusy = 0;
	135	ctx->queueEmpty = 1;
	136	{
	137	int error = 0;
	138	error \|= ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
	139	error \|= ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
	140	error \|= ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
	141	if (error) { POOL_free(ctx); return NULL; }
	142	}
	143	ctx->shutdown = 0;
	144	/* Allocate space for the thread handles */
	145	ctx->threads = (ZSTD_pthread_t)ZSTD_customMalloc(numThreads sizeof(ZSTD_pthread_t), customMem);
	146	ctx->threadCapacity = 0;
	147	ctx->customMem = customMem;
	148	/* Check for errors */
	149	if (!ctx->threads \|\| !ctx->queue) { POOL_free(ctx); return NULL; }
	150	/* Initialize the threads */
	151	{ size_t i;
	152	for (i = 0; i < numThreads; ++i) {
	153	if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
	154	ctx->threadCapacity = i;
	155	POOL_free(ctx);
	156	return NULL;
	157	} }
	158	ctx->threadCapacity = numThreads;
	159	ctx->threadLimit = numThreads;
	160	}
	161	return ctx;
	162	}
	163
	164	/*! POOL_join() :
	165	Shutdown the queue, wake any sleeping threads, and join all of the threads.
	166	*/
	167	static void POOL_join(POOL_ctx* ctx) {
	168	/* Shut down the queue */
	169	ZSTD_pthread_mutex_lock(&ctx->queueMutex);
	170	ctx->shutdown = 1;
	171	ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
	172	/* Wake up sleeping threads */
	173	ZSTD_pthread_cond_broadcast(&ctx->queuePushCond);
	174	ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
	175	/* Join all of the threads */
	176	{ size_t i;
	177	for (i = 0; i < ctx->threadCapacity; ++i) {
	178	ZSTD_pthread_join(ctx->threads[i], NULL); /* note : could fail */
	179	} }
	180	}
	181
	182	void POOL_free(POOL_ctx *ctx) {
	183	if (!ctx) { return; }
	184	POOL_join(ctx);
	185	ZSTD_pthread_mutex_destroy(&ctx->queueMutex);
	186	ZSTD_pthread_cond_destroy(&ctx->queuePushCond);
	187	ZSTD_pthread_cond_destroy(&ctx->queuePopCond);
	188	ZSTD_customFree(ctx->queue, ctx->customMem);
	189	ZSTD_customFree(ctx->threads, ctx->customMem);
	190	ZSTD_customFree(ctx, ctx->customMem);
	191	}
	192
	193	void ZSTD_freeThreadPool (ZSTD_threadPool* pool) {
	194	POOL_free (pool);
	195	}
	196
	197	size_t POOL_sizeof(const POOL_ctx* ctx) {
	198	if (ctx==NULL) return 0; /* supports sizeof NULL */
	199	return sizeof(*ctx)
	200	+ ctx->queueSize * sizeof(POOL_job)
	201	+ ctx->threadCapacity * sizeof(ZSTD_pthread_t);
	202	}
	203
	204
	205	/* @return : 0 on success, 1 on error */
	206	static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads)
	207	{
	208	if (numThreads <= ctx->threadCapacity) {
	209	if (!numThreads) return 1;
	210	ctx->threadLimit = numThreads;
	211	return 0;
	212	}
	213	/* numThreads > threadCapacity */
	214	{ ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t)ZSTD_customMalloc(numThreads sizeof(ZSTD_pthread_t), ctx->customMem);
	215	if (!threadPool) return 1;
	216	/* replace existing thread pool */
	217	ZSTD_memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool));
	218	ZSTD_customFree(ctx->threads, ctx->customMem);
	219	ctx->threads = threadPool;
	220	/* Initialize additional threads */
	221	{ size_t threadId;
	222	for (threadId = ctx->threadCapacity; threadId < numThreads; ++threadId) {
	223	if (ZSTD_pthread_create(&threadPool[threadId], NULL, &POOL_thread, ctx)) {
	224	ctx->threadCapacity = threadId;
	225	return 1;
	226	} }
	227	} }
	228	/* successfully expanded */
	229	ctx->threadCapacity = numThreads;
	230	ctx->threadLimit = numThreads;
	231	return 0;
	232	}
	233
	234	/* @return : 0 on success, 1 on error */
	235	int POOL_resize(POOL_ctx* ctx, size_t numThreads)
	236	{
	237	int result;
	238	if (ctx==NULL) return 1;
	239	ZSTD_pthread_mutex_lock(&ctx->queueMutex);
	240	result = POOL_resize_internal(ctx, numThreads);
	241	ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
	242	ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
	243	return result;
	244	}
	245
	246	/**
	247	* Returns 1 if the queue is full and 0 otherwise.
	248	*
	249	* When queueSize is 1 (pool was created with an intended queueSize of 0),
	250	* then a queue is empty if there is a thread free _and_ no job is waiting.
	251	*/
	252	static int isQueueFull(POOL_ctx const* ctx) {
	253	if (ctx->queueSize > 1) {
	254	return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize);
	255	} else {
	256	return (ctx->numThreadsBusy == ctx->threadLimit) \|\|
	257	!ctx->queueEmpty;
	258	}
	259	}
	260
	261
	262	static void
	263	POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque)
	264	{
	265	POOL_job const job = {function, opaque};
	266	assert(ctx != NULL);
	267	if (ctx->shutdown) return;
	268
	269	ctx->queueEmpty = 0;
	270	ctx->queue[ctx->queueTail] = job;
	271	ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize;
	272	ZSTD_pthread_cond_signal(&ctx->queuePopCond);
	273	}
	274
	275	void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque)
	276	{
	277	assert(ctx != NULL);
	278	ZSTD_pthread_mutex_lock(&ctx->queueMutex);
	279	/* Wait until there is space in the queue for the new job */
	280	while (isQueueFull(ctx) && (!ctx->shutdown)) {
	281	ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
	282	}
	283	POOL_add_internal(ctx, function, opaque);
	284	ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
	285	}
	286
	287
	288	int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque)
	289	{
	290	assert(ctx != NULL);
	291	ZSTD_pthread_mutex_lock(&ctx->queueMutex);
	292	if (isQueueFull(ctx)) {
	293	ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
	294	return 0;
	295	}
	296	POOL_add_internal(ctx, function, opaque);
	297	ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
	298	return 1;
	299	}
	300
	301
	302	#else /* ZSTD_MULTITHREAD not defined */
	303
	304	/* ========================== */
	305	/* No multi-threading support */
	306	/* ========================== */
	307
	308
	309	/* We don't need any data, but if it is empty, malloc() might return NULL. */
	310	struct POOL_ctx_s {
	311	int dummy;
	312	};
	313	static POOL_ctx g_poolCtx;
	314
	315	POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
	316	return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem);
	317	}
	318
	319	POOL_ctx*
	320	POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem)
	321	{
	322	(void)numThreads;
	323	(void)queueSize;
	324	(void)customMem;
	325	return &g_poolCtx;
	326	}
	327
	328	void POOL_free(POOL_ctx* ctx) {
	329	assert(!ctx \|\| ctx == &g_poolCtx);
	330	(void)ctx;
	331	}
	332
	333	int POOL_resize(POOL_ctx* ctx, size_t numThreads) {
	334	(void)ctx; (void)numThreads;
	335	return 0;
	336	}
	337
	338	void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) {
	339	(void)ctx;
	340	function(opaque);
	341	}
	342
	343	int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) {
	344	(void)ctx;
	345	function(opaque);
	346	return 1;
	347	}
	348
	349	size_t POOL_sizeof(const POOL_ctx* ctx) {
	350	if (ctx==NULL) return 0; /* supports sizeof NULL */
	351	assert(ctx == &g_poolCtx);
	352	return sizeof(*ctx);
	353	}
	354
	355	#endif /* ZSTD_MULTITHREAD */

stage1/zstd/lib/common/pool.h created+84

...	...	@@ -0,0 +1,84 @@
	1	/*
	2	* Copyright (c) Yann Collet, Facebook, Inc.
	3	* All rights reserved.
	4	*
	5	* This source code is licensed under both the BSD-style license (found in the
	6	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	7	* in the COPYING file in the root directory of this source tree).
	8	* You may select, at your option, one of the above-listed licenses.
	9	*/
	10
	11	#ifndef POOL_H
	12	#define POOL_H
	13
	14	#if defined (__cplusplus)
	15	extern "C" {
	16	#endif
	17
	18
	19	#include "zstd_deps.h"
	20	#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */
	21	#include "../zstd.h"
	22
	23	typedef struct POOL_ctx_s POOL_ctx;
	24
	25	/*! POOL_create() :
	26	* Create a thread pool with at most `numThreads` threads.
	27	* `numThreads` must be at least 1.
	28	* The maximum number of queued jobs before blocking is `queueSize`.
	29	* @return : POOL_ctx pointer on success, else NULL.
	30	*/
	31	POOL_ctx* POOL_create(size_t numThreads, size_t queueSize);
	32
	33	POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
	34	ZSTD_customMem customMem);
	35
	36	/*! POOL_free() :
	37	* Free a thread pool returned by POOL_create().
	38	*/
	39	void POOL_free(POOL_ctx* ctx);
	40
	41	/*! POOL_resize() :
	42	* Expands or shrinks pool's number of threads.
	43	* This is more efficient than releasing + creating a new context,
	44	* since it tries to preserve and re-use existing threads.
	45	* `numThreads` must be at least 1.
	46	* @return : 0 when resize was successful,
	47	* !0 (typically 1) if there is an error.
	48	* note : only numThreads can be resized, queueSize remains unchanged.
	49	*/
	50	int POOL_resize(POOL_ctx* ctx, size_t numThreads);
	51
	52	/*! POOL_sizeof() :
	53	* @return threadpool memory usage
	54	* note : compatible with NULL (returns 0 in this case)
	55	*/
	56	size_t POOL_sizeof(const POOL_ctx* ctx);
	57
	58	/*! POOL_function :
	59	* The function type that can be added to a thread pool.
	60	*/
	61	typedef void (POOL_function)(void);
	62
	63	/*! POOL_add() :
	64	* Add the job `function(opaque)` to the thread pool. `ctx` must be valid.
	65	* Possibly blocks until there is room in the queue.
	66	* Note : The function may be executed asynchronously,
	67	* therefore, `opaque` must live until function has been completed.
	68	*/
	69	void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque);
	70
	71
	72	/*! POOL_tryAdd() :
	73	* Add the job `function(opaque)` to thread pool _if_ a queue slot is available.
	74	* Returns immediately even if not (does not block).
	75	* @return : 1 if successful, 0 if not.
	76	*/
	77	int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque);
	78
	79
	80	#if defined (__cplusplus)
	81	}
	82	#endif
	83
	84	#endif

stage1/zstd/lib/common/portability_macros.h created+137

...	...	@@ -0,0 +1,137 @@
	1	/*
	2	* Copyright (c) Facebook, Inc.
	3	* All rights reserved.
	4	*
	5	* This source code is licensed under both the BSD-style license (found in the
	6	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	7	* in the COPYING file in the root directory of this source tree).
	8	* You may select, at your option, one of the above-listed licenses.
	9	*/
	10
	11	#ifndef ZSTD_PORTABILITY_MACROS_H
	12	#define ZSTD_PORTABILITY_MACROS_H
	13
	14	/**
	15	* This header file contains macro defintions to support portability.
	16	* This header is shared between C and ASM code, so it MUST only
	17	* contain macro definitions. It MUST not contain any C code.
	18	*
	19	* This header ONLY defines macros to detect platforms/feature support.
	20	*
	21	*/
	22
	23
	24	/* compat. with non-clang compilers */
	25	#ifndef __has_attribute
	26	#define __has_attribute(x) 0
	27	#endif
	28
	29	/* compat. with non-clang compilers */
	30	#ifndef __has_builtin
	31	# define __has_builtin(x) 0
	32	#endif
	33
	34	/* compat. with non-clang compilers */
	35	#ifndef __has_feature
	36	# define __has_feature(x) 0
	37	#endif
	38
	39	/* detects whether we are being compiled under msan */
	40	#ifndef ZSTD_MEMORY_SANITIZER
	41	# if __has_feature(memory_sanitizer)
	42	# define ZSTD_MEMORY_SANITIZER 1
	43	# else
	44	# define ZSTD_MEMORY_SANITIZER 0
	45	# endif
	46	#endif
	47
	48	/* detects whether we are being compiled under asan */
	49	#ifndef ZSTD_ADDRESS_SANITIZER
	50	# if __has_feature(address_sanitizer)
	51	# define ZSTD_ADDRESS_SANITIZER 1
	52	# elif defined(__SANITIZE_ADDRESS__)
	53	# define ZSTD_ADDRESS_SANITIZER 1
	54	# else
	55	# define ZSTD_ADDRESS_SANITIZER 0
	56	# endif
	57	#endif
	58
	59	/* detects whether we are being compiled under dfsan */
	60	#ifndef ZSTD_DATAFLOW_SANITIZER
	61	# if __has_feature(dataflow_sanitizer)
	62	# define ZSTD_DATAFLOW_SANITIZER 1
	63	# else
	64	# define ZSTD_DATAFLOW_SANITIZER 0
	65	# endif
	66	#endif
	67
	68	/* Mark the internal assembly functions as hidden */
	69	#ifdef __ELF__
	70	# define ZSTD_HIDE_ASM_FUNCTION(func) .hidden func
	71	#else
	72	# define ZSTD_HIDE_ASM_FUNCTION(func)
	73	#endif
	74
	75	/* Enable runtime BMI2 dispatch based on the CPU.
	76	* Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
	77	*/
	78	#ifndef DYNAMIC_BMI2
	79	#if ((defined(__clang__) && __has_attribute(__target__)) \
	80	\|\| (defined(__GNUC__) \
	81	&& (__GNUC__ >= 5 \|\| (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \
	82	&& (defined(__x86_64__) \|\| defined(_M_X64)) \
	83	&& !defined(__BMI2__)
	84	# define DYNAMIC_BMI2 1
	85	#else
	86	# define DYNAMIC_BMI2 0
	87	#endif
	88	#endif
	89
	90	/**
	91	* Only enable assembly for GNUC comptabile compilers,
	92	* because other platforms may not support GAS assembly syntax.
	93	*
	94	* Only enable assembly for Linux / MacOS, other platforms may
	95	* work, but they haven't been tested. This could likely be
	96	* extended to BSD systems.
	97	*
	98	* Disable assembly when MSAN is enabled, because MSAN requires
	99	* 100% of code to be instrumented to work.
	100	*/
	101	#if defined(__GNUC__)
	102	# if defined(__linux__) \|\| defined(__linux) \|\| defined(__APPLE__)
	103	# if ZSTD_MEMORY_SANITIZER
	104	# define ZSTD_ASM_SUPPORTED 0
	105	# elif ZSTD_DATAFLOW_SANITIZER
	106	# define ZSTD_ASM_SUPPORTED 0
	107	# else
	108	# define ZSTD_ASM_SUPPORTED 1
	109	# endif
	110	# else
	111	# define ZSTD_ASM_SUPPORTED 0
	112	# endif
	113	#else
	114	# define ZSTD_ASM_SUPPORTED 0
	115	#endif
	116
	117	/**
	118	* Determines whether we should enable assembly for x86-64
	119	* with BMI2.
	120	*
	121	* Enable if all of the following conditions hold:
	122	* - ASM hasn't been explicitly disabled by defining ZSTD_DISABLE_ASM
	123	* - Assembly is supported
	124	* - We are compiling for x86-64 and either:
	125	* - DYNAMIC_BMI2 is enabled
	126	* - BMI2 is supported at compile time
	127	*/
	128	#if !defined(ZSTD_DISABLE_ASM) && \
	129	ZSTD_ASM_SUPPORTED && \
	130	defined(__x86_64__) && \
	131	(DYNAMIC_BMI2 \|\| defined(__BMI2__))
	132	# define ZSTD_ENABLE_ASM_X86_64_BMI2 1
	133	#else
	134	# define ZSTD_ENABLE_ASM_X86_64_BMI2 0
	135	#endif
	136
	137	#endif /* ZSTD_PORTABILITY_MACROS_H */

stage1/zstd/lib/common/threading.c created+122

...	...	@@ -0,0 +1,122 @@
	1	/**
	2	* Copyright (c) 2016 Tino Reichardt
	3	* All rights reserved.
	4	*
	5	* You can contact the author at:
	6	* - zstdmt source repository: https://github.com/mcmilk/zstdmt
	7	*
	8	* This source code is licensed under both the BSD-style license (found in the
	9	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	10	* in the COPYING file in the root directory of this source tree).
	11	* You may select, at your option, one of the above-listed licenses.
	12	*/
	13
	14	/**
	15	* This file will hold wrapper for systems, which do not support pthreads
	16	*/
	17
	18	#include "threading.h"
	19
	20	/* create fake symbol to avoid empty translation unit warning */
	21	int g_ZSTD_threading_useless_symbol;
	22
	23	#if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
	24
	25	/**
	26	* Windows minimalist Pthread Wrapper, based on :
	27	* http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
	28	*/
	29
	30
	31	/* === Dependencies === */
	32	#include <process.h>
	33	#include <errno.h>
	34
	35
	36	/* === Implementation === */
	37
	38	static unsigned __stdcall worker(void *arg)
	39	{
	40	ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg;
	41	thread->arg = thread->start_routine(thread->arg);
	42	return 0;
	43	}
	44
	45	int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
	46	void* (start_routine) (void), void* arg)
	47	{
	48	(void)unused;
	49	thread->arg = arg;
	50	thread->start_routine = start_routine;
	51	thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL);
	52
	53	if (!thread->handle)
	54	return errno;
	55	else
	56	return 0;
	57	}
	58
	59	int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
	60	{
	61	DWORD result;
	62
	63	if (!thread.handle) return 0;
	64
	65	result = WaitForSingleObject(thread.handle, INFINITE);
	66	switch (result) {
	67	case WAIT_OBJECT_0:
	68	if (value_ptr) *value_ptr = thread.arg;
	69	return 0;
	70	case WAIT_ABANDONED:
	71	return EINVAL;
	72	default:
	73	return GetLastError();
	74	}
	75	}
	76
	77	#endif /* ZSTD_MULTITHREAD */
	78
	79	#if defined(ZSTD_MULTITHREAD) && DEBUGLEVEL >= 1 && !defined(_WIN32)
	80
	81	#define ZSTD_DEPS_NEED_MALLOC
	82	#include "zstd_deps.h"
	83
	84	int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr)
	85	{
	86	mutex = (pthread_mutex_t)ZSTD_malloc(sizeof(pthread_mutex_t));
	87	if (!*mutex)
	88	return 1;
	89	return pthread_mutex_init(*mutex, attr);
	90	}
	91
	92	int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex)
	93	{
	94	if (!*mutex)
	95	return 0;
	96	{
	97	int const ret = pthread_mutex_destroy(*mutex);
	98	ZSTD_free(*mutex);
	99	return ret;
	100	}
	101	}
	102
	103	int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr)
	104	{
	105	cond = (pthread_cond_t)ZSTD_malloc(sizeof(pthread_cond_t));
	106	if (!*cond)
	107	return 1;
	108	return pthread_cond_init(*cond, attr);
	109	}
	110
	111	int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond)
	112	{
	113	if (!*cond)
	114	return 0;
	115	{
	116	int const ret = pthread_cond_destroy(*cond);
	117	ZSTD_free(*cond);
	118	return ret;
	119	}
	120	}
	121
	122	#endif

stage1/zstd/lib/common/threading.h created+155

...	...	@@ -0,0 +1,155 @@
	1	/**
	2	* Copyright (c) 2016 Tino Reichardt
	3	* All rights reserved.
	4	*
	5	* You can contact the author at:
	6	* - zstdmt source repository: https://github.com/mcmilk/zstdmt
	7	*
	8	* This source code is licensed under both the BSD-style license (found in the
	9	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	10	* in the COPYING file in the root directory of this source tree).
	11	* You may select, at your option, one of the above-listed licenses.
	12	*/
	13
	14	#ifndef THREADING_H_938743
	15	#define THREADING_H_938743
	16
	17	#include "debug.h"
	18
	19	#if defined (__cplusplus)
	20	extern "C" {
	21	#endif
	22
	23	#if defined(ZSTD_MULTITHREAD) && defined(_WIN32)
	24
	25	/**
	26	* Windows minimalist Pthread Wrapper, based on :
	27	* http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
	28	*/
	29	#ifdef WINVER
	30	# undef WINVER
	31	#endif
	32	#define WINVER 0x0600
	33
	34	#ifdef _WIN32_WINNT
	35	# undef _WIN32_WINNT
	36	#endif
	37	#define _WIN32_WINNT 0x0600
	38
	39	#ifndef WIN32_LEAN_AND_MEAN
	40	# define WIN32_LEAN_AND_MEAN
	41	#endif
	42
	43	#undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */
	44	#include <windows.h>
	45	#undef ERROR
	46	#define ERROR(name) ZSTD_ERROR(name)
	47
	48
	49	/* mutex */
	50	#define ZSTD_pthread_mutex_t CRITICAL_SECTION
	51	#define ZSTD_pthread_mutex_init(a, b) ((void)(b), InitializeCriticalSection((a)), 0)
	52	#define ZSTD_pthread_mutex_destroy(a) DeleteCriticalSection((a))
	53	#define ZSTD_pthread_mutex_lock(a) EnterCriticalSection((a))
	54	#define ZSTD_pthread_mutex_unlock(a) LeaveCriticalSection((a))
	55
	56	/* condition variable */
	57	#define ZSTD_pthread_cond_t CONDITION_VARIABLE
	58	#define ZSTD_pthread_cond_init(a, b) ((void)(b), InitializeConditionVariable((a)), 0)
	59	#define ZSTD_pthread_cond_destroy(a) ((void)(a))
	60	#define ZSTD_pthread_cond_wait(a, b) SleepConditionVariableCS((a), (b), INFINITE)
	61	#define ZSTD_pthread_cond_signal(a) WakeConditionVariable((a))
	62	#define ZSTD_pthread_cond_broadcast(a) WakeAllConditionVariable((a))
	63
	64	/* ZSTD_pthread_create() and ZSTD_pthread_join() */
	65	typedef struct {
	66	HANDLE handle;
	67	void* (start_routine)(void);
	68	void* arg;
	69	} ZSTD_pthread_t;
	70
	71	int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
	72	void* (start_routine) (void), void* arg);
	73
	74	int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
	75
	76	/**
	77	* add here more wrappers as required
	78	*/
	79
	80
	81	#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */
	82	/* === POSIX Systems === */
	83	# include <pthread.h>
	84
	85	#if DEBUGLEVEL < 1
	86
	87	#define ZSTD_pthread_mutex_t pthread_mutex_t
	88	#define ZSTD_pthread_mutex_init(a, b) pthread_mutex_init((a), (b))
	89	#define ZSTD_pthread_mutex_destroy(a) pthread_mutex_destroy((a))
	90	#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock((a))
	91	#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock((a))
	92
	93	#define ZSTD_pthread_cond_t pthread_cond_t
	94	#define ZSTD_pthread_cond_init(a, b) pthread_cond_init((a), (b))
	95	#define ZSTD_pthread_cond_destroy(a) pthread_cond_destroy((a))
	96	#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait((a), (b))
	97	#define ZSTD_pthread_cond_signal(a) pthread_cond_signal((a))
	98	#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast((a))
	99
	100	#define ZSTD_pthread_t pthread_t
	101	#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
	102	#define ZSTD_pthread_join(a, b) pthread_join((a),(b))
	103
	104	#else /* DEBUGLEVEL >= 1 */
	105
	106	/* Debug implementation of threading.
	107	* In this implementation we use pointers for mutexes and condition variables.
	108	* This way, if we forget to init/destroy them the program will crash or ASAN
	109	* will report leaks.
	110	*/
	111
	112	#define ZSTD_pthread_mutex_t pthread_mutex_t*
	113	int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr);
	114	int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex);
	115	#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock(*(a))
	116	#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock(*(a))
	117
	118	#define ZSTD_pthread_cond_t pthread_cond_t*
	119	int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr);
	120	int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond);
	121	#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait((a), (b))
	122	#define ZSTD_pthread_cond_signal(a) pthread_cond_signal(*(a))
	123	#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast(*(a))
	124
	125	#define ZSTD_pthread_t pthread_t
	126	#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
	127	#define ZSTD_pthread_join(a, b) pthread_join((a),(b))
	128
	129	#endif
	130
	131	#else /* ZSTD_MULTITHREAD not defined */
	132	/* No multithreading support */
	133
	134	typedef int ZSTD_pthread_mutex_t;
	135	#define ZSTD_pthread_mutex_init(a, b) ((void)(a), (void)(b), 0)
	136	#define ZSTD_pthread_mutex_destroy(a) ((void)(a))
	137	#define ZSTD_pthread_mutex_lock(a) ((void)(a))
	138	#define ZSTD_pthread_mutex_unlock(a) ((void)(a))
	139
	140	typedef int ZSTD_pthread_cond_t;
	141	#define ZSTD_pthread_cond_init(a, b) ((void)(a), (void)(b), 0)
	142	#define ZSTD_pthread_cond_destroy(a) ((void)(a))
	143	#define ZSTD_pthread_cond_wait(a, b) ((void)(a), (void)(b))
	144	#define ZSTD_pthread_cond_signal(a) ((void)(a))
	145	#define ZSTD_pthread_cond_broadcast(a) ((void)(a))
	146
	147	/* do not use ZSTD_pthread_t */
	148
	149	#endif /* ZSTD_MULTITHREAD */
	150
	151	#if defined (__cplusplus)
	152	}
	153	#endif
	154
	155	#endif /* THREADING_H_938743 */

stage1/zstd/lib/common/xxhash.c created+24

...	...	@@ -0,0 +1,24 @@
	1	/*
	2	* xxHash - Fast Hash algorithm
	3	* Copyright (c) Yann Collet, Facebook, Inc.
	4	*
	5	* You can contact the author at :
	6	* - xxHash homepage: http://www.xxhash.com
	7	* - xxHash source repository : https://github.com/Cyan4973/xxHash
	8	*
	9	* This source code is licensed under both the BSD-style license (found in the
	10	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	11	* in the COPYING file in the root directory of this source tree).
	12	* You may select, at your option, one of the above-listed licenses.
	13	*/
	14
	15
	16
	17	/*
	18	* xxhash.c instantiates functions defined in xxhash.h
	19	*/
	20
	21	#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */
	22	#define XXH_IMPLEMENTATION /* access definitions */
	23
	24	#include "xxhash.h"

stage1/zstd/lib/common/xxhash.h created+5686

...	...	@@ -0,0 +1,5686 @@
	1	/*
	2	* xxHash - Fast Hash algorithm
	3	* Copyright (c) Yann Collet, Facebook, Inc.
	4	*
	5	* You can contact the author at :
	6	* - xxHash homepage: http://www.xxhash.com
	7	* - xxHash source repository : https://github.com/Cyan4973/xxHash
	8	*
	9	* This source code is licensed under both the BSD-style license (found in the
	10	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	11	* in the COPYING file in the root directory of this source tree).
	12	* You may select, at your option, one of the above-listed licenses.
	13	*/
	14
	15
	16	#ifndef XXH_NO_XXH3
	17	# define XXH_NO_XXH3
	18	#endif
	19
	20	#ifndef XXH_NAMESPACE
	21	# define XXH_NAMESPACE ZSTD_
	22	#endif
	23
	24	/*!
	25	* @mainpage xxHash
	26	*
	27	* @file xxhash.h
	28	* xxHash prototypes and implementation
	29	*/
	30	/* TODO: update */
	31	/* Notice extracted from xxHash homepage:
	32
	33	xxHash is an extremely fast hash algorithm, running at RAM speed limits.
	34	It also successfully passes all tests from the SMHasher suite.
	35
	36	Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
	37
	38	Name Speed Q.Score Author
	39	xxHash 5.4 GB/s 10
	40	CrapWow 3.2 GB/s 2 Andrew
	41	MurmurHash 3a 2.7 GB/s 10 Austin Appleby
	42	SpookyHash 2.0 GB/s 10 Bob Jenkins
	43	SBox 1.4 GB/s 9 Bret Mulvey
	44	Lookup3 1.2 GB/s 9 Bob Jenkins
	45	SuperFastHash 1.2 GB/s 1 Paul Hsieh
	46	CityHash64 1.05 GB/s 10 Pike & Alakuijala
	47	FNV 0.55 GB/s 5 Fowler, Noll, Vo
	48	CRC32 0.43 GB/s 9
	49	MD5-32 0.33 GB/s 10 Ronald L. Rivest
	50	SHA1-32 0.28 GB/s 10
	51
	52	Q.Score is a measure of quality of the hash function.
	53	It depends on successfully passing SMHasher test set.
	54	10 is a perfect score.
	55
	56	Note: SMHasher's CRC32 implementation is not the fastest one.
	57	Other speed-oriented implementations can be faster,
	58	especially in combination with PCLMUL instruction:
	59	https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html?showComment=1552696407071#c3490092340461170735
	60
	61	A 64-bit version, named XXH64, is available since r35.
	62	It offers much better speed, but for 64-bit applications only.
	63	Name Speed on 64 bits Speed on 32 bits
	64	XXH64 13.8 GB/s 1.9 GB/s
	65	XXH32 6.8 GB/s 6.0 GB/s
	66	*/
	67
	68	#if defined (__cplusplus)
	69	extern "C" {
	70	#endif
	71
	72	/* ****************************
	73	* INLINE mode
	74	******************************/
	75	/*!
	76	* XXH_INLINE_ALL (and XXH_PRIVATE_API)
	77	* Use these build macros to inline xxhash into the target unit.
	78	* Inlining improves performance on small inputs, especially when the length is
	79	* expressed as a compile-time constant:
	80	*
	81	* https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html
	82	*
	83	* It also keeps xxHash symbols private to the unit, so they are not exported.
	84	*
	85	* Usage:
	86	* #define XXH_INLINE_ALL
	87	* #include "xxhash.h"
	88	*
	89	* Do not compile and link xxhash.o as a separate object, as it is not useful.
	90	*/
	91	#if (defined(XXH_INLINE_ALL) \|\| defined(XXH_PRIVATE_API)) \
	92	&& !defined(XXH_INLINE_ALL_31684351384)
	93	/* this section should be traversed only once */
	94	# define XXH_INLINE_ALL_31684351384
	95	/* give access to the advanced API, required to compile implementations */
	96	# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */
	97	# define XXH_STATIC_LINKING_ONLY
	98	/* make all functions private */
	99	# undef XXH_PUBLIC_API
	100	# if defined(__GNUC__)
	101	# define XXH_PUBLIC_API static __inline __attribute__((unused))
	102	# elif defined (__cplusplus) \|\| (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
	103	# define XXH_PUBLIC_API static inline
	104	# elif defined(_MSC_VER)
	105	# define XXH_PUBLIC_API static __inline
	106	# else
	107	/* note: this version may generate warnings for unused static functions */
	108	# define XXH_PUBLIC_API static
	109	# endif
	110
	111	/*
	112	* This part deals with the special case where a unit wants to inline xxHash,
	113	* but "xxhash.h" has previously been included without XXH_INLINE_ALL,
	114	* such as part of some previously included *.h header file.
	115	* Without further action, the new include would just be ignored,
	116	* and functions would effectively _not_ be inlined (silent failure).
	117	* The following macros solve this situation by prefixing all inlined names,
	118	* avoiding naming collision with previous inclusions.
	119	*/
	120	/* Before that, we unconditionally #undef all symbols,
	121	* in case they were already defined with XXH_NAMESPACE.
	122	* They will then be redefined for XXH_INLINE_ALL
	123	*/
	124	# undef XXH_versionNumber
	125	/* XXH32 */
	126	# undef XXH32
	127	# undef XXH32_createState
	128	# undef XXH32_freeState
	129	# undef XXH32_reset
	130	# undef XXH32_update
	131	# undef XXH32_digest
	132	# undef XXH32_copyState
	133	# undef XXH32_canonicalFromHash
	134	# undef XXH32_hashFromCanonical
	135	/* XXH64 */
	136	# undef XXH64
	137	# undef XXH64_createState
	138	# undef XXH64_freeState
	139	# undef XXH64_reset
	140	# undef XXH64_update
	141	# undef XXH64_digest
	142	# undef XXH64_copyState
	143	# undef XXH64_canonicalFromHash
	144	# undef XXH64_hashFromCanonical
	145	/* XXH3_64bits */
	146	# undef XXH3_64bits
	147	# undef XXH3_64bits_withSecret
	148	# undef XXH3_64bits_withSeed
	149	# undef XXH3_64bits_withSecretandSeed
	150	# undef XXH3_createState
	151	# undef XXH3_freeState
	152	# undef XXH3_copyState
	153	# undef XXH3_64bits_reset
	154	# undef XXH3_64bits_reset_withSeed
	155	# undef XXH3_64bits_reset_withSecret
	156	# undef XXH3_64bits_update
	157	# undef XXH3_64bits_digest
	158	# undef XXH3_generateSecret
	159	/* XXH3_128bits */
	160	# undef XXH128
	161	# undef XXH3_128bits
	162	# undef XXH3_128bits_withSeed
	163	# undef XXH3_128bits_withSecret
	164	# undef XXH3_128bits_reset
	165	# undef XXH3_128bits_reset_withSeed
	166	# undef XXH3_128bits_reset_withSecret
	167	# undef XXH3_128bits_reset_withSecretandSeed
	168	# undef XXH3_128bits_update
	169	# undef XXH3_128bits_digest
	170	# undef XXH128_isEqual
	171	# undef XXH128_cmp
	172	# undef XXH128_canonicalFromHash
	173	# undef XXH128_hashFromCanonical
	174	/* Finally, free the namespace itself */
	175	# undef XXH_NAMESPACE
	176
	177	/* employ the namespace for XXH_INLINE_ALL */
	178	# define XXH_NAMESPACE XXH_INLINE_
	179	/*
	180	* Some identifiers (enums, type names) are not symbols,
	181	* but they must nonetheless be renamed to avoid redeclaration.
	182	* Alternative solution: do not redeclare them.
	183	* However, this requires some #ifdefs, and has a more dispersed impact.
	184	* Meanwhile, renaming can be achieved in a single place.
	185	*/
	186	# define XXH_IPREF(Id) XXH_NAMESPACE ## Id
	187	# define XXH_OK XXH_IPREF(XXH_OK)
	188	# define XXH_ERROR XXH_IPREF(XXH_ERROR)
	189	# define XXH_errorcode XXH_IPREF(XXH_errorcode)
	190	# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t)
	191	# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t)
	192	# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t)
	193	# define XXH32_state_s XXH_IPREF(XXH32_state_s)
	194	# define XXH32_state_t XXH_IPREF(XXH32_state_t)
	195	# define XXH64_state_s XXH_IPREF(XXH64_state_s)
	196	# define XXH64_state_t XXH_IPREF(XXH64_state_t)
	197	# define XXH3_state_s XXH_IPREF(XXH3_state_s)
	198	# define XXH3_state_t XXH_IPREF(XXH3_state_t)
	199	# define XXH128_hash_t XXH_IPREF(XXH128_hash_t)
	200	/* Ensure the header is parsed again, even if it was previously included */
	201	# undef XXHASH_H_5627135585666179
	202	# undef XXHASH_H_STATIC_13879238742
	203	#endif /* XXH_INLINE_ALL \|\| XXH_PRIVATE_API */
	204
	205
	206
	207	/* ****************************************************************
	208	* Stable API
	209	*****************************************************************/
	210	#ifndef XXHASH_H_5627135585666179
	211	#define XXHASH_H_5627135585666179 1
	212
	213
	214	/*!
	215	* @defgroup public Public API
	216	* Contains details on the public xxHash functions.
	217	* @{
	218	*/
	219	/* specific declaration modes for Windows */
	220	#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API)
	221	# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) \|\| defined(XXH_EXPORT))
	222	# ifdef XXH_EXPORT
	223	# define XXH_PUBLIC_API __declspec(dllexport)
	224	# elif XXH_IMPORT
	225	# define XXH_PUBLIC_API __declspec(dllimport)
	226	# endif
	227	# else
	228	# define XXH_PUBLIC_API /* do nothing */
	229	# endif
	230	#endif
	231
	232	#ifdef XXH_DOXYGEN
	233	/*!
	234	* @brief Emulate a namespace by transparently prefixing all symbols.
	235	*
	236	* If you want to include _and expose_ xxHash functions from within your own
	237	* library, but also want to avoid symbol collisions with other libraries which
	238	* may also include xxHash, you can use XXH_NAMESPACE to automatically prefix
	239	* any public symbol from xxhash library with the value of XXH_NAMESPACE
	240	* (therefore, avoid empty or numeric values).
	241	*
	242	* Note that no change is required within the calling program as long as it
	243	* includes `xxhash.h`: Regular symbol names will be automatically translated
	244	* by this header.
	245	*/
	246	# define XXH_NAMESPACE /* YOUR NAME HERE */
	247	# undef XXH_NAMESPACE
	248	#endif
	249
	250	#ifdef XXH_NAMESPACE
	251	# define XXH_CAT(A,B) A##B
	252	# define XXH_NAME2(A,B) XXH_CAT(A,B)
	253	# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
	254	/* XXH32 */
	255	# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
	256	# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
	257	# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
	258	# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
	259	# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
	260	# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
	261	# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
	262	# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
	263	# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
	264	/* XXH64 */
	265	# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
	266	# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
	267	# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
	268	# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
	269	# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
	270	# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
	271	# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
	272	# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
	273	# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
	274	/* XXH3_64bits */
	275	# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits)
	276	# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret)
	277	# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed)
	278	# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed)
	279	# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState)
	280	# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState)
	281	# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState)
	282	# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset)
	283	# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed)
	284	# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret)
	285	# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed)
	286	# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update)
	287	# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest)
	288	# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret)
	289	# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed)
	290	/* XXH3_128bits */
	291	# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128)
	292	# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits)
	293	# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed)
	294	# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret)
	295	# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed)
	296	# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset)
	297	# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed)
	298	# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret)
	299	# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed)
	300	# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update)
	301	# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest)
	302	# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual)
	303	# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp)
	304	# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash)
	305	# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical)
	306	#endif
	307
	308
	309	/* *************************************
	310	* Version
	311	***************************************/
	312	#define XXH_VERSION_MAJOR 0
	313	#define XXH_VERSION_MINOR 8
	314	#define XXH_VERSION_RELEASE 1
	315	#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR 100100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
	316
	317	/*!
	318	* @brief Obtains the xxHash version.
	319	*
	320	* This is mostly useful when xxHash is compiled as a shared library,
	321	* since the returned value comes from the library, as opposed to header file.
	322	*
	323	* @return `XXH_VERSION_NUMBER` of the invoked library.
	324	*/
	325	XXH_PUBLIC_API unsigned XXH_versionNumber (void);
	326
	327
	328	/* ****************************
	329	* Common basic types
	330	******************************/
	331	#include <stddef.h> /* size_t */
	332	typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
	333
	334
	335	/-*********************************************************************
	336	* 32-bit hash
	337	************************************************************************/
	338	#if defined(XXH_DOXYGEN) /* Don't show <stdint.h> include */
	339	/*!
	340	* @brief An unsigned 32-bit integer.
	341	*
	342	* Not necessarily defined to `uint32_t` but functionally equivalent.
	343	*/
	344	typedef uint32_t XXH32_hash_t;
	345
	346	#elif !defined (__VMS) \
	347	&& (defined (__cplusplus) \
	348	\|\| (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
	349	# include <stdint.h>
	350	typedef uint32_t XXH32_hash_t;
	351
	352	#else
	353	# include <limits.h>
	354	# if UINT_MAX == 0xFFFFFFFFUL
	355	typedef unsigned int XXH32_hash_t;
	356	# else
	357	# if ULONG_MAX == 0xFFFFFFFFUL
	358	typedef unsigned long XXH32_hash_t;
	359	# else
	360	# error "unsupported platform: need a 32-bit type"
	361	# endif
	362	# endif
	363	#endif
	364
	365	/*!
	366	* @}
	367	*
	368	* @defgroup xxh32_family XXH32 family
	369	* @ingroup public
	370	* Contains functions used in the classic 32-bit xxHash algorithm.
	371	*
	372	* @note
	373	* XXH32 is useful for older platforms, with no or poor 64-bit performance.
	374	* Note that @ref xxh3_family provides competitive speed
	375	* for both 32-bit and 64-bit systems, and offers true 64/128 bit hash results.
	376	*
	377	* @see @ref xxh64_family, @ref xxh3_family : Other xxHash families
	378	* @see @ref xxh32_impl for implementation details
	379	* @{
	380	*/
	381
	382	/*!
	383	* @brief Calculates the 32-bit hash of @p input using xxHash32.
	384	*
	385	* Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s
	386	*
	387	* @param input The block of data to be hashed, at least @p length bytes in size.
	388	* @param length The length of @p input, in bytes.
	389	* @param seed The 32-bit seed to alter the hash's output predictably.
	390	*
	391	* @pre
	392	* The memory between @p input and @p input + @p length must be valid,
	393	* readable, contiguous memory. However, if @p length is `0`, @p input may be
	394	* `NULL`. In C++, this also must be TriviallyCopyable.
	395	*
	396	* @return The calculated 32-bit hash value.
	397	*
	398	* @see
	399	* XXH64(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128():
	400	* Direct equivalents for the other variants of xxHash.
	401	* @see
	402	* XXH32_createState(), XXH32_update(), XXH32_digest(): Streaming version.
	403	*/
	404	XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed);
	405
	406	/*!
	407	* Streaming functions generate the xxHash value from an incremental input.
	408	* This method is slower than single-call functions, due to state management.
	409	* For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized.
	410	*
	411	* An XXH state must first be allocated using `XXH*_createState()`.
	412	*
	413	* Start a new hash by initializing the state with a seed using `XXH*_reset()`.
	414	*
	415	* Then, feed the hash state by calling `XXH*_update()` as many times as necessary.
	416	*
	417	* The function returns an error code, with 0 meaning OK, and any other value
	418	* meaning there is an error.
	419	*
	420	* Finally, a hash value can be produced anytime, by using `XXH*_digest()`.
	421	* This function returns the nn-bits hash as an int or long long.
	422	*
	423	* It's still possible to continue inserting input into the hash state after a
	424	* digest, and generate new hash values later on by invoking `XXH*_digest()`.
	425	*
	426	* When done, release the state using `XXH*_freeState()`.
	427	*
	428	* Example code for incrementally hashing a file:
	429	* @code{.c}
	430	* #include <stdio.h>
	431	* #include <xxhash.h>
	432	* #define BUFFER_SIZE 256
	433	*
	434	* // Note: XXH64 and XXH3 use the same interface.
	435	* XXH32_hash_t
	436	* hashFile(FILE* stream)
	437	* {
	438	* XXH32_state_t* state;
	439	* unsigned char buf[BUFFER_SIZE];
	440	* size_t amt;
	441	* XXH32_hash_t hash;
	442	*
	443	* state = XXH32_createState(); // Create a state
	444	* assert(state != NULL); // Error check here
	445	* XXH32_reset(state, 0xbaad5eed); // Reset state with our seed
	446	* while ((amt = fread(buf, 1, sizeof(buf), stream)) != 0) {
	447	* XXH32_update(state, buf, amt); // Hash the file in chunks
	448	* }
	449	* hash = XXH32_digest(state); // Finalize the hash
	450	* XXH32_freeState(state); // Clean up
	451	* return hash;
	452	* }
	453	* @endcode
	454	*/
	455
	456	/*!
	457	* @typedef struct XXH32_state_s XXH32_state_t
	458	* @brief The opaque state struct for the XXH32 streaming API.
	459	*
	460	* @see XXH32_state_s for details.
	461	*/
	462	typedef struct XXH32_state_s XXH32_state_t;
	463
	464	/*!
	465	* @brief Allocates an @ref XXH32_state_t.
	466	*
	467	* Must be freed with XXH32_freeState().
	468	* @return An allocated XXH32_state_t on success, `NULL` on failure.
	469	*/
	470	XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);
	471	/*!
	472	* @brief Frees an @ref XXH32_state_t.
	473	*
	474	* Must be allocated with XXH32_createState().
	475	* @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState().
	476	* @return XXH_OK.
	477	*/
	478	XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);
	479	/*!
	480	* @brief Copies one @ref XXH32_state_t to another.
	481	*
	482	* @param dst_state The state to copy to.
	483	* @param src_state The state to copy from.
	484	* @pre
	485	* @p dst_state and @p src_state must not be `NULL` and must not overlap.
	486	*/
	487	XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state);
	488
	489	/*!
	490	* @brief Resets an @ref XXH32_state_t to begin a new hash.
	491	*
	492	* This function resets and seeds a state. Call it before @ref XXH32_update().
	493	*
	494	* @param statePtr The state struct to reset.
	495	* @param seed The 32-bit seed to alter the hash result predictably.
	496	*
	497	* @pre
	498	* @p statePtr must not be `NULL`.
	499	*
	500	* @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
	501	*/
	502	XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed);
	503
	504	/*!
	505	* @brief Consumes a block of @p input to an @ref XXH32_state_t.
	506	*
	507	* Call this to incrementally consume blocks of data.
	508	*
	509	* @param statePtr The state struct to update.
	510	* @param input The block of data to be hashed, at least @p length bytes in size.
	511	* @param length The length of @p input, in bytes.
	512	*
	513	* @pre
	514	* @p statePtr must not be `NULL`.
	515	* @pre
	516	* The memory between @p input and @p input + @p length must be valid,
	517	* readable, contiguous memory. However, if @p length is `0`, @p input may be
	518	* `NULL`. In C++, this also must be TriviallyCopyable.
	519	*
	520	* @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
	521	*/
	522	XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
	523
	524	/*!
	525	* @brief Returns the calculated hash value from an @ref XXH32_state_t.
	526	*
	527	* @note
	528	* Calling XXH32_digest() will not affect @p statePtr, so you can update,
	529	* digest, and update again.
	530	*
	531	* @param statePtr The state struct to calculate the hash from.
	532	*
	533	* @pre
	534	* @p statePtr must not be `NULL`.
	535	*
	536	* @return The calculated xxHash32 value from that state.
	537	*/
	538	XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr);
	539
	540	/***** Canonical representation *****/
	541
	542	/*
	543	* The default return values from XXH functions are unsigned 32 and 64 bit
	544	* integers.
	545	* This the simplest and fastest format for further post-processing.
	546	*
	547	* However, this leaves open the question of what is the order on the byte level,
	548	* since little and big endian conventions will store the same number differently.
	549	*
	550	* The canonical representation settles this issue by mandating big-endian
	551	* convention, the same convention as human-readable numbers (large digits first).
	552	*
	553	* When writing hash values to storage, sending them over a network, or printing
	554	* them, it's highly recommended to use the canonical representation to ensure
	555	* portability across a wider range of systems, present and future.
	556	*
	557	* The following functions allow transformation of hash values to and from
	558	* canonical format.
	559	*/
	560
	561	/*!
	562	* @brief Canonical (big endian) representation of @ref XXH32_hash_t.
	563	*/
	564	typedef struct {
	565	unsigned char digest[4]; /!< Hash bytes, big endian /
	566	} XXH32_canonical_t;
	567
	568	/*!
	569	* @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t.
	570	*
	571	* @param dst The @ref XXH32_canonical_t pointer to be stored to.
	572	* @param hash The @ref XXH32_hash_t to be converted.
	573	*
	574	* @pre
	575	* @p dst must not be `NULL`.
	576	*/
	577	XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
	578
	579	/*!
	580	* @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t.
	581	*
	582	* @param src The @ref XXH32_canonical_t to convert.
	583	*
	584	* @pre
	585	* @p src must not be `NULL`.
	586	*
	587	* @return The converted hash.
	588	*/
	589	XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
	590
	591
	592	#ifdef __has_attribute
	593	# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x)
	594	#else
	595	# define XXH_HAS_ATTRIBUTE(x) 0
	596	#endif
	597
	598	/* C-language Attributes are added in C23. */
	599	#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute)
	600	# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x)
	601	#else
	602	# define XXH_HAS_C_ATTRIBUTE(x) 0
	603	#endif
	604
	605	#if defined(__cplusplus) && defined(__has_cpp_attribute)
	606	# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
	607	#else
	608	# define XXH_HAS_CPP_ATTRIBUTE(x) 0
	609	#endif
	610
	611	/*
	612	Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute
	613	introduced in CPP17 and C23.
	614	CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough
	615	C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough
	616	*/
	617	#if XXH_HAS_C_ATTRIBUTE(x)
	618	# define XXH_FALLTHROUGH [[fallthrough]]
	619	#elif XXH_HAS_CPP_ATTRIBUTE(x)
	620	# define XXH_FALLTHROUGH [[fallthrough]]
	621	#elif XXH_HAS_ATTRIBUTE(__fallthrough__)
	622	# define XXH_FALLTHROUGH __attribute__ ((fallthrough))
	623	#else
	624	# define XXH_FALLTHROUGH
	625	#endif
	626
	627	/*!
	628	* @}
	629	* @ingroup public
	630	* @{
	631	*/
	632
	633	#ifndef XXH_NO_LONG_LONG
	634	/-*********************************************************************
	635	* 64-bit hash
	636	************************************************************************/
	637	#if defined(XXH_DOXYGEN) /* don't include <stdint.h> */
	638	/*!
	639	* @brief An unsigned 64-bit integer.
	640	*
	641	* Not necessarily defined to `uint64_t` but functionally equivalent.
	642	*/
	643	typedef uint64_t XXH64_hash_t;
	644	#elif !defined (__VMS) \
	645	&& (defined (__cplusplus) \
	646	\|\| (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
	647	# include <stdint.h>
	648	typedef uint64_t XXH64_hash_t;
	649	#else
	650	# include <limits.h>
	651	# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL
	652	/* LP64 ABI says uint64_t is unsigned long */
	653	typedef unsigned long XXH64_hash_t;
	654	# else
	655	/* the following type must have a width of 64-bit */
	656	typedef unsigned long long XXH64_hash_t;
	657	# endif
	658	#endif
	659
	660	/*!
	661	* @}
	662	*
	663	* @defgroup xxh64_family XXH64 family
	664	* @ingroup public
	665	* @{
	666	* Contains functions used in the classic 64-bit xxHash algorithm.
	667	*
	668	* @note
	669	* XXH3 provides competitive speed for both 32-bit and 64-bit systems,
	670	* and offers true 64/128 bit hash results.
	671	* It provides better speed for systems with vector processing capabilities.
	672	*/
	673
	674
	675	/*!
	676	* @brief Calculates the 64-bit hash of @p input using xxHash64.
	677	*
	678	* This function usually runs faster on 64-bit systems, but slower on 32-bit
	679	* systems (see benchmark).
	680	*
	681	* @param input The block of data to be hashed, at least @p length bytes in size.
	682	* @param length The length of @p input, in bytes.
	683	* @param seed The 64-bit seed to alter the hash's output predictably.
	684	*
	685	* @pre
	686	* The memory between @p input and @p input + @p length must be valid,
	687	* readable, contiguous memory. However, if @p length is `0`, @p input may be
	688	* `NULL`. In C++, this also must be TriviallyCopyable.
	689	*
	690	* @return The calculated 64-bit hash.
	691	*
	692	* @see
	693	* XXH32(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128():
	694	* Direct equivalents for the other variants of xxHash.
	695	* @see
	696	* XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version.
	697	*/
	698	XXH_PUBLIC_API XXH64_hash_t XXH64(const void* input, size_t length, XXH64_hash_t seed);
	699
	700	/***** Streaming *****/
	701	/*!
	702	* @brief The opaque state struct for the XXH64 streaming API.
	703	*
	704	* @see XXH64_state_s for details.
	705	*/
	706	typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */
	707	XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void);
	708	XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);
	709	XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state);
	710
	711	XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, XXH64_hash_t seed);
	712	XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
	713	XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr);
	714
	715	/***** Canonical representation *****/
	716	typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t;
	717	XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash);
	718	XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
	719
	720	#ifndef XXH_NO_XXH3
	721	/*!
	722	* @}
	723	* ************************************************************************
	724	* @defgroup xxh3_family XXH3 family
	725	* @ingroup public
	726	* @{
	727	*
	728	* XXH3 is a more recent hash algorithm featuring:
	729	* - Improved speed for both small and large inputs
	730	* - True 64-bit and 128-bit outputs
	731	* - SIMD acceleration
	732	* - Improved 32-bit viability
	733	*
	734	* Speed analysis methodology is explained here:
	735	*
	736	* https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html
	737	*
	738	* Compared to XXH64, expect XXH3 to run approximately
	739	* ~2x faster on large inputs and >3x faster on small ones,
	740	* exact differences vary depending on platform.
	741	*
	742	* XXH3's speed benefits greatly from SIMD and 64-bit arithmetic,
	743	* but does not require it.
	744	* Any 32-bit and 64-bit targets that can run XXH32 smoothly
	745	* can run XXH3 at competitive speeds, even without vector support.
	746	* Further details are explained in the implementation.
	747	*
	748	* Optimized implementations are provided for AVX512, AVX2, SSE2, NEON, POWER8,
	749	* ZVector and scalar targets. This can be controlled via the XXH_VECTOR macro.
	750	*
	751	* XXH3 implementation is portable:
	752	* it has a generic C90 formulation that can be compiled on any platform,
	753	* all implementations generage exactly the same hash value on all platforms.
	754	* Starting from v0.8.0, it's also labelled "stable", meaning that
	755	* any future version will also generate the same hash value.
	756	*
	757	* XXH3 offers 2 variants, _64bits and _128bits.
	758	*
	759	* When only 64 bits are needed, prefer invoking the _64bits variant, as it
	760	* reduces the amount of mixing, resulting in faster speed on small inputs.
	761	* It's also generally simpler to manipulate a scalar return type than a struct.
	762	*
	763	* The API supports one-shot hashing, streaming mode, and custom secrets.
	764	*/
	765
	766	/-*********************************************************************
	767	* XXH3 64-bit variant
	768	************************************************************************/
	769
	770	/* XXH3_64bits():
	771	* default 64-bit variant, using default secret and default seed of 0.
	772	* It's the fastest variant. */
	773	XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* data, size_t len);
	774
	775	/*
	776	* XXH3_64bits_withSeed():
	777	* This variant generates a custom secret on the fly
	778	* based on default secret altered using the `seed` value.
	779	* While this operation is decently fast, note that it's not completely free.
	780	* Note: seed==0 produces the same results as XXH3_64bits().
	781	*/
	782	XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void* data, size_t len, XXH64_hash_t seed);
	783
	784	/*!
	785	* The bare minimum size for a custom secret.
	786	*
	787	* @see
	788	* XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(),
	789	* XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret().
	790	*/
	791	#define XXH3_SECRET_SIZE_MIN 136
	792
	793	/*
	794	* XXH3_64bits_withSecret():
	795	* It's possible to provide any blob of bytes as a "secret" to generate the hash.
	796	* This makes it more difficult for an external actor to prepare an intentional collision.
	797	* The main condition is that secretSize must be large enough (>= XXH3_SECRET_SIZE_MIN).
	798	* However, the quality of the secret impacts the dispersion of the hash algorithm.
	799	* Therefore, the secret _must_ look like a bunch of random bytes.
	800	* Avoid "trivial" or structured data such as repeated sequences or a text document.
	801	* Whenever in doubt about the "randomness" of the blob of bytes,
	802	* consider employing "XXH3_generateSecret()" instead (see below).
	803	* It will generate a proper high entropy secret derived from the blob of bytes.
	804	* Another advantage of using XXH3_generateSecret() is that
	805	* it guarantees that all bits within the initial blob of bytes
	806	* will impact every bit of the output.
	807	* This is not necessarily the case when using the blob of bytes directly
	808	* because, when hashing _small_ inputs, only a portion of the secret is employed.
	809	*/
	810	XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize);
	811
	812
	813	/***** Streaming *****/
	814	/*
	815	* Streaming requires state maintenance.
	816	* This operation costs memory and CPU.
	817	* As a consequence, streaming is slower than one-shot hashing.
	818	* For better performance, prefer one-shot functions whenever applicable.
	819	*/
	820
	821	/*!
	822	* @brief The state struct for the XXH3 streaming API.
	823	*
	824	* @see XXH3_state_s for details.
	825	*/
	826	typedef struct XXH3_state_s XXH3_state_t;
	827	XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void);
	828	XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr);
	829	XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state);
	830
	831	/*
	832	* XXH3_64bits_reset():
	833	* Initialize with default parameters.
	834	* digest will be equivalent to `XXH3_64bits()`.
	835	*/
	836	XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t* statePtr);
	837	/*
	838	* XXH3_64bits_reset_withSeed():
	839	* Generate a custom secret from `seed`, and store it into `statePtr`.
	840	* digest will be equivalent to `XXH3_64bits_withSeed()`.
	841	*/
	842	XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed);
	843	/*
	844	* XXH3_64bits_reset_withSecret():
	845	* `secret` is referenced, it _must outlive_ the hash streaming session.
	846	* Similar to one-shot API, `secretSize` must be >= `XXH3_SECRET_SIZE_MIN`,
	847	* and the quality of produced hash values depends on secret's entropy
	848	* (secret's content should look like a bunch of random bytes).
	849	* When in doubt about the randomness of a candidate `secret`,
	850	* consider employing `XXH3_generateSecret()` instead (see below).
	851	*/
	852	XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize);
	853
	854	XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH3_state_t* statePtr, const void* input, size_t length);
	855	XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* statePtr);
	856
	857	/* note : canonical representation of XXH3 is the same as XXH64
	858	* since they both produce XXH64_hash_t values */
	859
	860
	861	/-*********************************************************************
	862	* XXH3 128-bit variant
	863	************************************************************************/
	864
	865	/*!
	866	* @brief The return value from 128-bit hashes.
	867	*
	868	* Stored in little endian order, although the fields themselves are in native
	869	* endianness.
	870	*/
	871	typedef struct {
	872	XXH64_hash_t low64; /!< `value & 0xFFFFFFFFFFFFFFFF` /
	873	XXH64_hash_t high64; /!< `value >> 64` /
	874	} XXH128_hash_t;
	875
	876	XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* data, size_t len);
	877	XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void* data, size_t len, XXH64_hash_t seed);
	878	XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize);
	879
	880	/***** Streaming *****/
	881	/*
	882	* Streaming requires state maintenance.
	883	* This operation costs memory and CPU.
	884	* As a consequence, streaming is slower than one-shot hashing.
	885	* For better performance, prefer one-shot functions whenever applicable.
	886	*
	887	* XXH3_128bits uses the same XXH3_state_t as XXH3_64bits().
	888	* Use already declared XXH3_createState() and XXH3_freeState().
	889	*
	890	* All reset and streaming functions have same meaning as their 64-bit counterpart.
	891	*/
	892
	893	XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH3_state_t* statePtr);
	894	XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed);
	895	XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize);
	896
	897	XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH3_state_t* statePtr, const void* input, size_t length);
	898	XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* statePtr);
	899
	900	/* Following helper functions make it possible to compare XXH128_hast_t values.
	901	* Since XXH128_hash_t is a structure, this capability is not offered by the language.
	902	* Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */
	903
	904	/*!
	905	* XXH128_isEqual():
	906	* Return: 1 if `h1` and `h2` are equal, 0 if they are not.
	907	*/
	908	XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2);
	909
	910	/*!
	911	* XXH128_cmp():
	912	*
	913	* This comparator is compatible with stdlib's `qsort()`/`bsearch()`.
	914	*
	915	* return: >0 if h128_1 > h128_2
	916	* =0 if h128_1 == h128_2
	917	* <0 if h128_1 < h128_2
	918	*/
	919	XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2);
	920
	921
	922	/***** Canonical representation *****/
	923	typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t;
	924	XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash);
	925	XXH_PUBLIC_API XXH128_hash_t XXH128_hashFromCanonical(const XXH128_canonical_t* src);
	926
	927
	928	#endif /* !XXH_NO_XXH3 */
	929	#endif /* XXH_NO_LONG_LONG */
	930
	931	/*!
	932	* @}
	933	*/
	934	#endif /* XXHASH_H_5627135585666179 */
	935
	936
	937
	938	#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742)
	939	#define XXHASH_H_STATIC_13879238742
	940	/* ****************************************************************************
	941	* This section contains declarations which are not guaranteed to remain stable.
	942	* They may change in future versions, becoming incompatible with a different
	943	* version of the library.
	944	* These declarations should only be used with static linking.
	945	* Never use them in association with dynamic linking!
	946	***************************************************************************** */
	947
	948	/*
	949	* These definitions are only present to allow static allocation
	950	* of XXH states, on stack or in a struct, for example.
	951	* Never ever access their members directly.
	952	*/
	953
	954	/*!
	955	* @internal
	956	* @brief Structure for XXH32 streaming API.
	957	*
	958	* @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
	959	* @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is
	960	* an opaque type. This allows fields to safely be changed.
	961	*
	962	* Typedef'd to @ref XXH32_state_t.
	963	* Do not access the members of this struct directly.
	964	* @see XXH64_state_s, XXH3_state_s
	965	*/
	966	struct XXH32_state_s {
	967	XXH32_hash_t total_len_32; /!< Total length hashed, modulo 2^32 /
	968	XXH32_hash_t large_len; /!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) /
	969	XXH32_hash_t v[4]; /!< Accumulator lanes /
	970	XXH32_hash_t mem32[4]; /!< Internal buffer for partial reads. Treated as unsigned char[16]. /
	971	XXH32_hash_t memsize; /!< Amount of data in @ref mem32 /
	972	XXH32_hash_t reserved; /!< Reserved field. Do not read nor write to it. /
	973	}; /* typedef'd to XXH32_state_t */
	974
	975
	976	#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */
	977
	978	/*!
	979	* @internal
	980	* @brief Structure for XXH64 streaming API.
	981	*
	982	* @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
	983	* @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is
	984	* an opaque type. This allows fields to safely be changed.
	985	*
	986	* Typedef'd to @ref XXH64_state_t.
	987	* Do not access the members of this struct directly.
	988	* @see XXH32_state_s, XXH3_state_s
	989	*/
	990	struct XXH64_state_s {
	991	XXH64_hash_t total_len; /!< Total length hashed. This is always 64-bit. /
	992	XXH64_hash_t v[4]; /!< Accumulator lanes /
	993	XXH64_hash_t mem64[4]; /!< Internal buffer for partial reads. Treated as unsigned char[32]. /
	994	XXH32_hash_t memsize; /!< Amount of data in @ref mem64 /
	995	XXH32_hash_t reserved32; /!< Reserved field, needed for padding anyways/
	996	XXH64_hash_t reserved64; /!< Reserved field. Do not read or write to it. /
	997	}; /* typedef'd to XXH64_state_t */
	998
	999
	1000	#ifndef XXH_NO_XXH3
	1001
	1002	#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */
	1003	# include <stdalign.h>
	1004	# define XXH_ALIGN(n) alignas(n)
	1005	#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */
	1006	/* In C++ alignas() is a keyword */
	1007	# define XXH_ALIGN(n) alignas(n)
	1008	#elif defined(__GNUC__)
	1009	# define XXH_ALIGN(n) __attribute__ ((aligned(n)))
	1010	#elif defined(_MSC_VER)
	1011	# define XXH_ALIGN(n) __declspec(align(n))
	1012	#else
	1013	# define XXH_ALIGN(n) /* disabled */
	1014	#endif
	1015
	1016	/* Old GCC versions only accept the attribute after the type in structures. */
	1017	#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \
	1018	&& ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \
	1019	&& defined(__GNUC__)
	1020	# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align)
	1021	#else
	1022	# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type
	1023	#endif
	1024
	1025	/*!
	1026	* @brief The size of the internal XXH3 buffer.
	1027	*
	1028	* This is the optimal update size for incremental hashing.
	1029	*
	1030	* @see XXH3_64b_update(), XXH3_128b_update().
	1031	*/
	1032	#define XXH3_INTERNALBUFFER_SIZE 256
	1033
	1034	/*!
	1035	* @brief Default size of the secret buffer (and @ref XXH3_kSecret).
	1036	*
	1037	* This is the size used in @ref XXH3_kSecret and the seeded functions.
	1038	*
	1039	* Not to be confused with @ref XXH3_SECRET_SIZE_MIN.
	1040	*/
	1041	#define XXH3_SECRET_DEFAULT_SIZE 192
	1042
	1043	/*!
	1044	* @internal
	1045	* @brief Structure for XXH3 streaming API.
	1046	*
	1047	* @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
	1048	* @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined.
	1049	* Otherwise it is an opaque type.
	1050	* Never use this definition in combination with dynamic library.
	1051	* This allows fields to safely be changed in the future.
	1052	*
	1053	* @note This structure has a strict alignment requirement of 64 bytes!!
	1054	* Do not allocate this with `malloc()` or `new`,
	1055	* it will not be sufficiently aligned.
	1056	* Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation.
	1057	*
	1058	* Typedef'd to @ref XXH3_state_t.
	1059	* Do never access the members of this struct directly.
	1060	*
	1061	* @see XXH3_INITSTATE() for stack initialization.
	1062	* @see XXH3_createState(), XXH3_freeState().
	1063	* @see XXH32_state_s, XXH64_state_s
	1064	*/
	1065	struct XXH3_state_s {
	1066	XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]);
	1067	/!< The 8 accumulators. Similar to `vN` in @ref XXH32_state_s::v1 and @ref XXH64_state_s /
	1068	XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]);
	1069	/!< Used to store a custom secret generated from a seed. /
	1070	XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]);
	1071	/!< The internal buffer. @see XXH32_state_s::mem32 /
	1072	XXH32_hash_t bufferedSize;
	1073	/!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize /
	1074	XXH32_hash_t useSeed;
	1075	/!< Reserved field. Needed for padding on 64-bit. /
	1076	size_t nbStripesSoFar;
	1077	/!< Number or stripes processed. /
	1078	XXH64_hash_t totalLen;
	1079	/!< Total length hashed. 64-bit even on 32-bit targets. /
	1080	size_t nbStripesPerBlock;
	1081	/!< Number of stripes per block. /
	1082	size_t secretLimit;
	1083	/!< Size of @ref customSecret or @ref extSecret /
	1084	XXH64_hash_t seed;
	1085	/!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() /
	1086	XXH64_hash_t reserved64;
	1087	/!< Reserved field. /
	1088	const unsigned char* extSecret;
	1089	/*!< Reference to an external secret for the _withSecret variants, NULL
	1090	* for other variants. */
	1091	/* note: there may be some padding at the end due to alignment on 64 bytes */
	1092	}; /* typedef'd to XXH3_state_t */
	1093
	1094	#undef XXH_ALIGN_MEMBER
	1095
	1096	/*!
	1097	* @brief Initializes a stack-allocated `XXH3_state_s`.
	1098	*
	1099	* When the @ref XXH3_state_t structure is merely emplaced on stack,
	1100	* it should be initialized with XXH3_INITSTATE() or a memset()
	1101	* in case its first reset uses XXH3_NNbits_reset_withSeed().
	1102	* This init can be omitted if the first reset uses default or _withSecret mode.
	1103	* This operation isn't necessary when the state is created with XXH3_createState().
	1104	* Note that this doesn't prepare the state for a streaming operation,
	1105	* it's still necessary to use XXH3_NNbits_reset*() afterwards.
	1106	*/
	1107	#define XXH3_INITSTATE(XXH3_state_ptr) { (XXH3_state_ptr)->seed = 0; }
	1108
	1109
	1110	/* XXH128() :
	1111	* simple alias to pre-selected XXH3_128bits variant
	1112	*/
	1113	XXH_PUBLIC_API XXH128_hash_t XXH128(const void* data, size_t len, XXH64_hash_t seed);
	1114
	1115
	1116	/* === Experimental API === */
	1117	/* Symbols defined below must be considered tied to a specific library version. */
	1118
	1119	/*
	1120	* XXH3_generateSecret():
	1121	*
	1122	* Derive a high-entropy secret from any user-defined content, named customSeed.
	1123	* The generated secret can be used in combination with `*_withSecret()` functions.
	1124	* The `_withSecret()` variants are useful to provide a higher level of protection than 64-bit seed,
	1125	* as it becomes much more difficult for an external actor to guess how to impact the calculation logic.
	1126	*
	1127	* The function accepts as input a custom seed of any length and any content,
	1128	* and derives from it a high-entropy secret of length @secretSize
	1129	* into an already allocated buffer @secretBuffer.
	1130	* @secretSize must be >= XXH3_SECRET_SIZE_MIN
	1131	*
	1132	* The generated secret can then be used with any `*_withSecret()` variant.
	1133	* Functions `XXH3_128bits_withSecret()`, `XXH3_64bits_withSecret()`,
	1134	* `XXH3_128bits_reset_withSecret()` and `XXH3_64bits_reset_withSecret()`
	1135	* are part of this list. They all accept a `secret` parameter
	1136	* which must be large enough for implementation reasons (>= XXH3_SECRET_SIZE_MIN)
	1137	* _and_ feature very high entropy (consist of random-looking bytes).
	1138	* These conditions can be a high bar to meet, so
	1139	* XXH3_generateSecret() can be employed to ensure proper quality.
	1140	*
	1141	* customSeed can be anything. It can have any size, even small ones,
	1142	* and its content can be anything, even "poor entropy" sources such as a bunch of zeroes.
	1143	* The resulting `secret` will nonetheless provide all required qualities.
	1144	*
	1145	* When customSeedSize > 0, supplying NULL as customSeed is undefined behavior.
	1146	*/
	1147	XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(void* secretBuffer, size_t secretSize, const void* customSeed, size_t customSeedSize);
	1148
	1149
	1150	/*
	1151	* XXH3_generateSecret_fromSeed():
	1152	*
	1153	* Generate the same secret as the _withSeed() variants.
	1154	*
	1155	* The resulting secret has a length of XXH3_SECRET_DEFAULT_SIZE (necessarily).
	1156	* @secretBuffer must be already allocated, of size at least XXH3_SECRET_DEFAULT_SIZE bytes.
	1157	*
	1158	* The generated secret can be used in combination with
	1159	`_withSecret()` and `_withSecretandSeed()` variants.
	1160	* This generator is notably useful in combination with `_withSecretandSeed()`,
	1161	* as a way to emulate a faster `_withSeed()` variant.
	1162	*/
	1163	XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(void* secretBuffer, XXH64_hash_t seed);
	1164
	1165	/*
	1166	* *_withSecretandSeed() :
	1167	* These variants generate hash values using either
	1168	* @seed for "short" keys (< XXH3_MIDSIZE_MAX = 240 bytes)
	1169	* or @secret for "large" keys (>= XXH3_MIDSIZE_MAX).
	1170	*
	1171	* This generally benefits speed, compared to `_withSeed()` or `_withSecret()`.
	1172	* `_withSeed()` has to generate the secret on the fly for "large" keys.
	1173	* It's fast, but can be perceptible for "not so large" keys (< 1 KB).
	1174	* `_withSecret()` has to generate the masks on the fly for "small" keys,
	1175	* which requires more instructions than _withSeed() variants.
	1176	* Therefore, _withSecretandSeed variant combines the best of both worlds.
	1177	*
	1178	* When @secret has been generated by XXH3_generateSecret_fromSeed(),
	1179	* this variant produces exactly the same results as `_withSeed()` variant,
	1180	* hence offering only a pure speed benefit on "large" input,
	1181	* by skipping the need to regenerate the secret for every large input.
	1182	*
	1183	* Another usage scenario is to hash the secret to a 64-bit hash value,
	1184	* for example with XXH3_64bits(), which then becomes the seed,
	1185	* and then employ both the seed and the secret in _withSecretandSeed().
	1186	* On top of speed, an added benefit is that each bit in the secret
	1187	* has a 50% chance to swap each bit in the output,
	1188	* via its impact to the seed.
	1189	* This is not guaranteed when using the secret directly in "small data" scenarios,
	1190	* because only portions of the secret are employed for small data.
	1191	*/
	1192	XXH_PUBLIC_API XXH64_hash_t
	1193	XXH3_64bits_withSecretandSeed(const void* data, size_t len,
	1194	const void* secret, size_t secretSize,
	1195	XXH64_hash_t seed);
	1196
	1197	XXH_PUBLIC_API XXH128_hash_t
	1198	XXH3_128bits_withSecretandSeed(const void* data, size_t len,
	1199	const void* secret, size_t secretSize,
	1200	XXH64_hash_t seed64);
	1201
	1202	XXH_PUBLIC_API XXH_errorcode
	1203	XXH3_64bits_reset_withSecretandSeed(XXH3_state_t* statePtr,
	1204	const void* secret, size_t secretSize,
	1205	XXH64_hash_t seed64);
	1206
	1207	XXH_PUBLIC_API XXH_errorcode
	1208	XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr,
	1209	const void* secret, size_t secretSize,
	1210	XXH64_hash_t seed64);
	1211
	1212
	1213	#endif /* XXH_NO_XXH3 */
	1214	#endif /* XXH_NO_LONG_LONG */
	1215	#if defined(XXH_INLINE_ALL) \|\| defined(XXH_PRIVATE_API)
	1216	# define XXH_IMPLEMENTATION
	1217	#endif
	1218
	1219	#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */
	1220
	1221
	1222	/* ======================================================================== */
	1223	/* ======================================================================== */
	1224	/* ======================================================================== */
	1225
	1226
	1227	/-*********************************************************************
	1228	* xxHash implementation
	1229	-*********************************************************************
	1230	* xxHash's implementation used to be hosted inside xxhash.c.
	1231	*
	1232	* However, inlining requires implementation to be visible to the compiler,
	1233	* hence be included alongside the header.
	1234	* Previously, implementation was hosted inside xxhash.c,
	1235	* which was then #included when inlining was activated.
	1236	* This construction created issues with a few build and install systems,
	1237	* as it required xxhash.c to be stored in /include directory.
	1238	*
	1239	* xxHash implementation is now directly integrated within xxhash.h.
	1240	* As a consequence, xxhash.c is no longer needed in /include.
	1241	*
	1242	* xxhash.c is still available and is still useful.
	1243	* In a "normal" setup, when xxhash is not inlined,
	1244	* xxhash.h only exposes the prototypes and public symbols,
	1245	* while xxhash.c can be built into an object file xxhash.o
	1246	* which can then be linked into the final binary.
	1247	************************************************************************/
	1248
	1249	#if ( defined(XXH_INLINE_ALL) \|\| defined(XXH_PRIVATE_API) \
	1250	\|\| defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387)
	1251	# define XXH_IMPLEM_13a8737387
	1252
	1253	/* *************************************
	1254	* Tuning parameters
	1255	***************************************/
	1256
	1257	/*!
	1258	* @defgroup tuning Tuning parameters
	1259	* @{
	1260	*
	1261	* Various macros to control xxHash's behavior.
	1262	*/
	1263	#ifdef XXH_DOXYGEN
	1264	/*!
	1265	* @brief Define this to disable 64-bit code.
	1266	*
	1267	* Useful if only using the @ref xxh32_family and you have a strict C90 compiler.
	1268	*/
	1269	# define XXH_NO_LONG_LONG
	1270	# undef XXH_NO_LONG_LONG /* don't actually */
	1271	/*!
	1272	* @brief Controls how unaligned memory is accessed.
	1273	*
	1274	* By default, access to unaligned memory is controlled by `memcpy()`, which is
	1275	* safe and portable.
	1276	*
	1277	* Unfortunately, on some target/compiler combinations, the generated assembly
	1278	* is sub-optimal.
	1279	*
	1280	* The below switch allow selection of a different access method
	1281	* in the search for improved performance.
	1282	*
	1283	* @par Possible options:
	1284	*
	1285	* - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy`
	1286	* @par
	1287	* Use `memcpy()`. Safe and portable. Note that most modern compilers will
	1288	* eliminate the function call and treat it as an unaligned access.
	1289	*
	1290	* - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((packed))`
	1291	* @par
	1292	* Depends on compiler extensions and is therefore not portable.
	1293	* This method is safe _if_ your compiler supports it,
	1294	* and generally as fast or faster than `memcpy`.
	1295	*
	1296	* - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast
	1297	* @par
	1298	* Casts directly and dereferences. This method doesn't depend on the
	1299	* compiler, but it violates the C standard as it directly dereferences an
	1300	* unaligned pointer. It can generate buggy code on targets which do not
	1301	* support unaligned memory accesses, but in some circumstances, it's the
	1302	* only known way to get the most performance.
	1303	*
	1304	* - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift
	1305	* @par
	1306	* Also portable. This can generate the best code on old compilers which don't
	1307	* inline small `memcpy()` calls, and it might also be faster on big-endian
	1308	* systems which lack a native byteswap instruction. However, some compilers
	1309	* will emit literal byteshifts even if the target supports unaligned access.
	1310	* .
	1311	*
	1312	* @warning
	1313	* Methods 1 and 2 rely on implementation-defined behavior. Use these with
	1314	* care, as what works on one compiler/platform/optimization level may cause
	1315	* another to read garbage data or even crash.
	1316	*
	1317	* See http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details.
	1318	*
	1319	* Prefer these methods in priority order (0 > 3 > 1 > 2)
	1320	*/
	1321	# define XXH_FORCE_MEMORY_ACCESS 0
	1322
	1323	/*!
	1324	* @def XXH_FORCE_ALIGN_CHECK
	1325	* @brief If defined to non-zero, adds a special path for aligned inputs (XXH32()
	1326	* and XXH64() only).
	1327	*
	1328	* This is an important performance trick for architectures without decent
	1329	* unaligned memory access performance.
	1330	*
	1331	* It checks for input alignment, and when conditions are met, uses a "fast
	1332	* path" employing direct 32-bit/64-bit reads, resulting in _dramatically
	1333	* faster_ read speed.
	1334	*
	1335	* The check costs one initial branch per hash, which is generally negligible,
	1336	* but not zero.
	1337	*
	1338	* Moreover, it's not useful to generate an additional code path if memory
	1339	* access uses the same instruction for both aligned and unaligned
	1340	* addresses (e.g. x86 and aarch64).
	1341	*
	1342	* In these cases, the alignment check can be removed by setting this macro to 0.
	1343	* Then the code will always use unaligned memory access.
	1344	* Align check is automatically disabled on x86, x64 & arm64,
	1345	* which are platforms known to offer good unaligned memory accesses performance.
	1346	*
	1347	* This option does not affect XXH3 (only XXH32 and XXH64).
	1348	*/
	1349	# define XXH_FORCE_ALIGN_CHECK 0
	1350
	1351	/*!
	1352	* @def XXH_NO_INLINE_HINTS
	1353	* @brief When non-zero, sets all functions to `static`.
	1354	*
	1355	* By default, xxHash tries to force the compiler to inline almost all internal
	1356	* functions.
	1357	*
	1358	* This can usually improve performance due to reduced jumping and improved
	1359	* constant folding, but significantly increases the size of the binary which
	1360	* might not be favorable.
	1361	*
	1362	* Additionally, sometimes the forced inlining can be detrimental to performance,
	1363	* depending on the architecture.
	1364	*
	1365	* XXH_NO_INLINE_HINTS marks all internal functions as static, giving the
	1366	* compiler full control on whether to inline or not.
	1367	*
	1368	* When not optimizing (-O0), optimizing for size (-Os, -Oz), or using
	1369	* -fno-inline with GCC or Clang, this will automatically be defined.
	1370	*/
	1371	# define XXH_NO_INLINE_HINTS 0
	1372
	1373	/*!
	1374	* @def XXH32_ENDJMP
	1375	* @brief Whether to use a jump for `XXH32_finalize`.
	1376	*
	1377	* For performance, `XXH32_finalize` uses multiple branches in the finalizer.
	1378	* This is generally preferable for performance,
	1379	* but depending on exact architecture, a jmp may be preferable.
	1380	*
	1381	* This setting is only possibly making a difference for very small inputs.
	1382	*/
	1383	# define XXH32_ENDJMP 0
	1384
	1385	/*!
	1386	* @internal
	1387	* @brief Redefines old internal names.
	1388	*
	1389	* For compatibility with code that uses xxHash's internals before the names
	1390	* were changed to improve namespacing. There is no other reason to use this.
	1391	*/
	1392	# define XXH_OLD_NAMES
	1393	# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */
	1394	#endif /* XXH_DOXYGEN */
	1395	/*!
	1396	* @}
	1397	*/
	1398
	1399	#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
	1400	/* prefer __packed__ structures (method 1) for gcc on armv7+ and mips */
	1401	# if !defined(__clang__) && \
	1402	( \
	1403	(defined(__INTEL_COMPILER) && !defined(_WIN32)) \|\| \
	1404	( \
	1405	defined(__GNUC__) && ( \
	1406	(defined(__ARM_ARCH) && __ARM_ARCH >= 7) \|\| \
	1407	( \
	1408	defined(__mips__) && \
	1409	(__mips <= 5 \|\| __mips_isa_rev < 6) && \
	1410	(!defined(__mips16) \|\| defined(__mips_mips16e2)) \
	1411	) \
	1412	) \
	1413	) \
	1414	)
	1415	# define XXH_FORCE_MEMORY_ACCESS 1
	1416	# endif
	1417	#endif
	1418
	1419	#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
	1420	# if defined(__i386) \|\| defined(__x86_64__) \|\| defined(__aarch64__) \
	1421	\|\| defined(_M_IX86) \|\| defined(_M_X64) \|\| defined(_M_ARM64) /* visual */
	1422	# define XXH_FORCE_ALIGN_CHECK 0
	1423	# else
	1424	# define XXH_FORCE_ALIGN_CHECK 1
	1425	# endif
	1426	#endif
	1427
	1428	#ifndef XXH_NO_INLINE_HINTS
	1429	# if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ \
	1430	\|\| defined(__NO_INLINE__) /* -O0, -fno-inline */
	1431	# define XXH_NO_INLINE_HINTS 1
	1432	# else
	1433	# define XXH_NO_INLINE_HINTS 0
	1434	# endif
	1435	#endif
	1436
	1437	#ifndef XXH32_ENDJMP
	1438	/* generally preferable for performance */
	1439	# define XXH32_ENDJMP 0
	1440	#endif
	1441
	1442	/*!
	1443	* @defgroup impl Implementation
	1444	* @{
	1445	*/
	1446
	1447
	1448	/* *************************************
	1449	* Includes & Memory related functions
	1450	***************************************/
	1451	/* Modify the local functions below should you wish to use some other memory routines */
	1452	/* for ZSTD_malloc(), ZSTD_free() */
	1453	#define ZSTD_DEPS_NEED_MALLOC
	1454	#include "zstd_deps.h" /* size_t, ZSTD_malloc, ZSTD_free, ZSTD_memcpy */
	1455	static void* XXH_malloc(size_t s) { return ZSTD_malloc(s); }
	1456	static void XXH_free (void* p) { ZSTD_free(p); }
	1457	static void* XXH_memcpy(void* dest, const void* src, size_t size) { return ZSTD_memcpy(dest,src,size); }
	1458
	1459
	1460	/* *************************************
	1461	* Compiler Specific Options
	1462	***************************************/
	1463	#ifdef _MSC_VER /* Visual Studio warning fix */
	1464	# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
	1465	#endif
	1466
	1467	#if XXH_NO_INLINE_HINTS /* disable inlining hints */
	1468	# if defined(__GNUC__) \|\| defined(__clang__)
	1469	# define XXH_FORCE_INLINE static __attribute__((unused))
	1470	# else
	1471	# define XXH_FORCE_INLINE static
	1472	# endif
	1473	# define XXH_NO_INLINE static
	1474	/* enable inlining hints */
	1475	#elif defined(__GNUC__) \|\| defined(__clang__)
	1476	# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused))
	1477	# define XXH_NO_INLINE static __attribute__((noinline))
	1478	#elif defined(_MSC_VER) /* Visual Studio */
	1479	# define XXH_FORCE_INLINE static __forceinline
	1480	# define XXH_NO_INLINE static __declspec(noinline)
	1481	#elif defined (__cplusplus) \
	1482	\|\| (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */
	1483	# define XXH_FORCE_INLINE static inline
	1484	# define XXH_NO_INLINE static
	1485	#else
	1486	# define XXH_FORCE_INLINE static
	1487	# define XXH_NO_INLINE static
	1488	#endif
	1489
	1490
	1491
	1492	/* *************************************
	1493	* Debug
	1494	***************************************/
	1495	/*!
	1496	* @ingroup tuning
	1497	* @def XXH_DEBUGLEVEL
	1498	* @brief Sets the debugging level.
	1499	*
	1500	* XXH_DEBUGLEVEL is expected to be defined externally, typically via the
	1501	* compiler's command line options. The value must be a number.
	1502	*/
	1503	#ifndef XXH_DEBUGLEVEL
	1504	# ifdef DEBUGLEVEL /* backwards compat */
	1505	# define XXH_DEBUGLEVEL DEBUGLEVEL
	1506	# else
	1507	# define XXH_DEBUGLEVEL 0
	1508	# endif
	1509	#endif
	1510
	1511	#if (XXH_DEBUGLEVEL>=1)
	1512	# include <assert.h> /* note: can still be disabled with NDEBUG */
	1513	# define XXH_ASSERT(c) assert(c)
	1514	#else
	1515	# define XXH_ASSERT(c) ((void)0)
	1516	#endif
	1517
	1518	/* note: use after variable declarations */
	1519	#ifndef XXH_STATIC_ASSERT
	1520	# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */
	1521	# include <assert.h>
	1522	# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0)
	1523	# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */
	1524	# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0)
	1525	# else
	1526	# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0)
	1527	# endif
	1528	# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c)
	1529	#endif
	1530
	1531	/*!
	1532	* @internal
	1533	* @def XXH_COMPILER_GUARD(var)
	1534	* @brief Used to prevent unwanted optimizations for @p var.
	1535	*
	1536	* It uses an empty GCC inline assembly statement with a register constraint
	1537	* which forces @p var into a general purpose register (eg eax, ebx, ecx
	1538	* on x86) and marks it as modified.
	1539	*
	1540	* This is used in a few places to avoid unwanted autovectorization (e.g.
	1541	* XXH32_round()). All vectorization we want is explicit via intrinsics,
	1542	* and _usually_ isn't wanted elsewhere.
	1543	*
	1544	* We also use it to prevent unwanted constant folding for AArch64 in
	1545	* XXH3_initCustomSecret_scalar().
	1546	*/
	1547	#if defined(__GNUC__) \|\| defined(__clang__)
	1548	# define XXH_COMPILER_GUARD(var) __asm__ __volatile__("" : "+r" (var))
	1549	#else
	1550	# define XXH_COMPILER_GUARD(var) ((void)0)
	1551	#endif
	1552
	1553	/* *************************************
	1554	* Basic Types
	1555	***************************************/
	1556	#if !defined (__VMS) \
	1557	&& (defined (__cplusplus) \
	1558	\|\| (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
	1559	# include <stdint.h>
	1560	typedef uint8_t xxh_u8;
	1561	#else
	1562	typedef unsigned char xxh_u8;
	1563	#endif
	1564	typedef XXH32_hash_t xxh_u32;
	1565
	1566	#ifdef XXH_OLD_NAMES
	1567	# define BYTE xxh_u8
	1568	# define U8 xxh_u8
	1569	# define U32 xxh_u32
	1570	#endif
	1571
	1572	/* * Memory access * */
	1573
	1574	/*!
	1575	* @internal
	1576	* @fn xxh_u32 XXH_read32(const void* ptr)
	1577	* @brief Reads an unaligned 32-bit integer from @p ptr in native endianness.
	1578	*
	1579	* Affected by @ref XXH_FORCE_MEMORY_ACCESS.
	1580	*
	1581	* @param ptr The pointer to read from.
	1582	* @return The 32-bit native endian integer from the bytes at @p ptr.
	1583	*/
	1584
	1585	/*!
	1586	* @internal
	1587	* @fn xxh_u32 XXH_readLE32(const void* ptr)
	1588	* @brief Reads an unaligned 32-bit little endian integer from @p ptr.
	1589	*
	1590	* Affected by @ref XXH_FORCE_MEMORY_ACCESS.
	1591	*
	1592	* @param ptr The pointer to read from.
	1593	* @return The 32-bit little endian integer from the bytes at @p ptr.
	1594	*/
	1595
	1596	/*!
	1597	* @internal
	1598	* @fn xxh_u32 XXH_readBE32(const void* ptr)
	1599	* @brief Reads an unaligned 32-bit big endian integer from @p ptr.
	1600	*
	1601	* Affected by @ref XXH_FORCE_MEMORY_ACCESS.
	1602	*
	1603	* @param ptr The pointer to read from.
	1604	* @return The 32-bit big endian integer from the bytes at @p ptr.
	1605	*/
	1606
	1607	/*!
	1608	* @internal
	1609	* @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align)
	1610	* @brief Like @ref XXH_readLE32(), but has an option for aligned reads.
	1611	*
	1612	* Affected by @ref XXH_FORCE_MEMORY_ACCESS.
	1613	* Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is
	1614	* always @ref XXH_alignment::XXH_unaligned.
	1615	*
	1616	* @param ptr The pointer to read from.
	1617	* @param align Whether @p ptr is aligned.
	1618	* @pre
	1619	* If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte
	1620	* aligned.
	1621	* @return The 32-bit little endian integer from the bytes at @p ptr.
	1622	*/
	1623
	1624	#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
	1625	/*
	1626	* Manual byteshift. Best for old compilers which don't inline memcpy.
	1627	* We actually directly use XXH_readLE32 and XXH_readBE32.
	1628	*/
	1629	#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
	1630
	1631	/*
	1632	* Force direct memory access. Only works on CPU which support unaligned memory
	1633	* access in hardware.
	1634	*/
	1635	static xxh_u32 XXH_read32(const void* memPtr) { return (const xxh_u32) memPtr; }
	1636
	1637	#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
	1638
	1639	/*
	1640	* __pack instructions are safer but compiler specific, hence potentially
	1641	* problematic for some compilers.
	1642	*
	1643	* Currently only defined for GCC and ICC.
	1644	*/
	1645	#ifdef XXH_OLD_NAMES
	1646	typedef union { xxh_u32 u32; } __attribute__((packed)) unalign;
	1647	#endif
	1648	static xxh_u32 XXH_read32(const void* ptr)
	1649	{
	1650	typedef union { xxh_u32 u32; } __attribute__((packed)) xxh_unalign;
	1651	return ((const xxh_unalign*)ptr)->u32;
	1652	}
	1653
	1654	#else
	1655
	1656	/*
	1657	* Portable and safe solution. Generally efficient.
	1658	* see: http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
	1659	*/
	1660	static xxh_u32 XXH_read32(const void* memPtr)
	1661	{
	1662	xxh_u32 val;
	1663	XXH_memcpy(&val, memPtr, sizeof(val));
	1664	return val;
	1665	}
	1666
	1667	#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
	1668
	1669
	1670	/* * Endianness * */
	1671
	1672	/*!
	1673	* @ingroup tuning
	1674	* @def XXH_CPU_LITTLE_ENDIAN
	1675	* @brief Whether the target is little endian.
	1676	*
	1677	* Defined to 1 if the target is little endian, or 0 if it is big endian.
	1678	* It can be defined externally, for example on the compiler command line.
	1679	*
	1680	* If it is not defined,
	1681	* a runtime check (which is usually constant folded) is used instead.
	1682	*
	1683	* @note
	1684	* This is not necessarily defined to an integer constant.
	1685	*
	1686	* @see XXH_isLittleEndian() for the runtime check.
	1687	*/
	1688	#ifndef XXH_CPU_LITTLE_ENDIAN
	1689	/*
	1690	* Try to detect endianness automatically, to avoid the nonstandard behavior
	1691	* in `XXH_isLittleEndian()`
	1692	*/
	1693	# if defined(_WIN32) /* Windows is always little endian */ \
	1694	\|\| defined(__LITTLE_ENDIAN__) \
	1695	\|\| (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
	1696	# define XXH_CPU_LITTLE_ENDIAN 1
	1697	# elif defined(__BIG_ENDIAN__) \
	1698	\|\| (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
	1699	# define XXH_CPU_LITTLE_ENDIAN 0
	1700	# else
	1701	/*!
	1702	* @internal
	1703	* @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN.
	1704	*
	1705	* Most compilers will constant fold this.
	1706	*/
	1707	static int XXH_isLittleEndian(void)
	1708	{
	1709	/*
	1710	* Portable and well-defined behavior.
	1711	* Don't use static: it is detrimental to performance.
	1712	*/
	1713	const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 };
	1714	return one.c[0];
	1715	}
	1716	# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian()
	1717	# endif
	1718	#endif
	1719
	1720
	1721
	1722
	1723	/* ****************************************
	1724	* Compiler-specific Functions and Macros
	1725	******************************************/
	1726	#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
	1727
	1728	#ifdef __has_builtin
	1729	# define XXH_HAS_BUILTIN(x) __has_builtin(x)
	1730	#else
	1731	# define XXH_HAS_BUILTIN(x) 0
	1732	#endif
	1733
	1734	/*!
	1735	* @internal
	1736	* @def XXH_rotl32(x,r)
	1737	* @brief 32-bit rotate left.
	1738	*
	1739	* @param x The 32-bit integer to be rotated.
	1740	* @param r The number of bits to rotate.
	1741	* @pre
	1742	* @p r > 0 && @p r < 32
	1743	* @note
	1744	* @p x and @p r may be evaluated multiple times.
	1745	* @return The rotated result.
	1746	*/
	1747	#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \
	1748	&& XXH_HAS_BUILTIN(__builtin_rotateleft64)
	1749	# define XXH_rotl32 __builtin_rotateleft32
	1750	# define XXH_rotl64 __builtin_rotateleft64
	1751	/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */
	1752	#elif defined(_MSC_VER)
	1753	# define XXH_rotl32(x,r) _rotl(x,r)
	1754	# define XXH_rotl64(x,r) _rotl64(x,r)
	1755	#else
	1756	# define XXH_rotl32(x,r) (((x) << (r)) \| ((x) >> (32 - (r))))
	1757	# define XXH_rotl64(x,r) (((x) << (r)) \| ((x) >> (64 - (r))))
	1758	#endif
	1759
	1760	/*!
	1761	* @internal
	1762	* @fn xxh_u32 XXH_swap32(xxh_u32 x)
	1763	* @brief A 32-bit byteswap.
	1764	*
	1765	* @param x The 32-bit integer to byteswap.
	1766	* @return @p x, byteswapped.
	1767	*/
	1768	#if defined(_MSC_VER) /* Visual Studio */
	1769	# define XXH_swap32 _byteswap_ulong
	1770	#elif XXH_GCC_VERSION >= 403
	1771	# define XXH_swap32 __builtin_bswap32
	1772	#else
	1773	static xxh_u32 XXH_swap32 (xxh_u32 x)
	1774	{
	1775	return ((x << 24) & 0xff000000 ) \|
	1776	((x << 8) & 0x00ff0000 ) \|
	1777	((x >> 8) & 0x0000ff00 ) \|
	1778	((x >> 24) & 0x000000ff );
	1779	}
	1780	#endif
	1781
	1782
	1783	/* ***************************
	1784	* Memory reads
	1785	*****************************/
	1786
	1787	/*!
	1788	* @internal
	1789	* @brief Enum to indicate whether a pointer is aligned.
	1790	*/
	1791	typedef enum {
	1792	XXH_aligned, /!< Aligned /
	1793	XXH_unaligned /!< Possibly unaligned /
	1794	} XXH_alignment;
	1795
	1796	/*
	1797	* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load.
	1798	*
	1799	* This is ideal for older compilers which don't inline memcpy.
	1800	*/
	1801	#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
	1802
	1803	XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr)
	1804	{
	1805	const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
	1806	return bytePtr[0]
	1807	\| ((xxh_u32)bytePtr[1] << 8)
	1808	\| ((xxh_u32)bytePtr[2] << 16)
	1809	\| ((xxh_u32)bytePtr[3] << 24);
	1810	}
	1811
	1812	XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr)
	1813	{
	1814	const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
	1815	return bytePtr[3]
	1816	\| ((xxh_u32)bytePtr[2] << 8)
	1817	\| ((xxh_u32)bytePtr[1] << 16)
	1818	\| ((xxh_u32)bytePtr[0] << 24);
	1819	}
	1820
	1821	#else
	1822	XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr)
	1823	{
	1824	return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
	1825	}
	1826
	1827	static xxh_u32 XXH_readBE32(const void* ptr)
	1828	{
	1829	return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
	1830	}
	1831	#endif
	1832
	1833	XXH_FORCE_INLINE xxh_u32
	1834	XXH_readLE32_align(const void* ptr, XXH_alignment align)
	1835	{
	1836	if (align==XXH_unaligned) {
	1837	return XXH_readLE32(ptr);
	1838	} else {
	1839	return XXH_CPU_LITTLE_ENDIAN ? (const xxh_u32)ptr : XXH_swap32((const xxh_u32)ptr);
	1840	}
	1841	}
	1842
	1843
	1844	/* *************************************
	1845	* Misc
	1846	***************************************/
	1847	/! @ingroup public /
	1848	XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
	1849
	1850
	1851	/* *******************************************************************
	1852	* 32-bit hash functions
	1853	*********************************************************************/
	1854	/*!
	1855	* @}
	1856	* @defgroup xxh32_impl XXH32 implementation
	1857	* @ingroup impl
	1858	* @{
	1859	*/
	1860	/* #define instead of static const, to be used as initializers */
	1861	#define XXH_PRIME32_1 0x9E3779B1U /!< 0b10011110001101110111100110110001 /
	1862	#define XXH_PRIME32_2 0x85EBCA77U /!< 0b10000101111010111100101001110111 /
	1863	#define XXH_PRIME32_3 0xC2B2AE3DU /!< 0b11000010101100101010111000111101 /
	1864	#define XXH_PRIME32_4 0x27D4EB2FU /!< 0b00100111110101001110101100101111 /
	1865	#define XXH_PRIME32_5 0x165667B1U /!< 0b00010110010101100110011110110001 /
	1866
	1867	#ifdef XXH_OLD_NAMES
	1868	# define PRIME32_1 XXH_PRIME32_1
	1869	# define PRIME32_2 XXH_PRIME32_2
	1870	# define PRIME32_3 XXH_PRIME32_3
	1871	# define PRIME32_4 XXH_PRIME32_4
	1872	# define PRIME32_5 XXH_PRIME32_5
	1873	#endif
	1874
	1875	/*!
	1876	* @internal
	1877	* @brief Normal stripe processing routine.
	1878	*
	1879	* This shuffles the bits so that any bit from @p input impacts several bits in
	1880	* @p acc.
	1881	*
	1882	* @param acc The accumulator lane.
	1883	* @param input The stripe of input to mix.
	1884	* @return The mixed accumulator lane.
	1885	*/
	1886	static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input)
	1887	{
	1888	acc += input * XXH_PRIME32_2;
	1889	acc = XXH_rotl32(acc, 13);
	1890	acc *= XXH_PRIME32_1;
	1891	#if (defined(__SSE4_1__) \|\| defined(__aarch64__)) && !defined(XXH_ENABLE_AUTOVECTORIZE)
	1892	/*
	1893	* UGLY HACK:
	1894	* A compiler fence is the only thing that prevents GCC and Clang from
	1895	* autovectorizing the XXH32 loop (pragmas and attributes don't work for some
	1896	* reason) without globally disabling SSE4.1.
	1897	*
	1898	* The reason we want to avoid vectorization is because despite working on
	1899	* 4 integers at a time, there are multiple factors slowing XXH32 down on
	1900	* SSE4:
	1901	* - There's a ridiculous amount of lag from pmulld (10 cycles of latency on
	1902	* newer chips!) making it slightly slower to multiply four integers at
	1903	* once compared to four integers independently. Even when pmulld was
	1904	* fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE
	1905	* just to multiply unless doing a long operation.
	1906	*
	1907	* - Four instructions are required to rotate,
	1908	* movqda tmp, v // not required with VEX encoding
	1909	* pslld tmp, 13 // tmp <<= 13
	1910	* psrld v, 19 // x >>= 19
	1911	* por v, tmp // x \|= tmp
	1912	* compared to one for scalar:
	1913	* roll v, 13 // reliably fast across the board
	1914	* shldl v, v, 13 // Sandy Bridge and later prefer this for some reason
	1915	*
	1916	* - Instruction level parallelism is actually more beneficial here because
	1917	* the SIMD actually serializes this operation: While v1 is rotating, v2
	1918	* can load data, while v3 can multiply. SSE forces them to operate
	1919	* together.
	1920	*
	1921	* This is also enabled on AArch64, as Clang autovectorizes it incorrectly
	1922	* and it is pointless writing a NEON implementation that is basically the
	1923	* same speed as scalar for XXH32.
	1924	*/
	1925	XXH_COMPILER_GUARD(acc);
	1926	#endif
	1927	return acc;
	1928	}
	1929
	1930	/*!
	1931	* @internal
	1932	* @brief Mixes all bits to finalize the hash.
	1933	*
	1934	* The final mix ensures that all input bits have a chance to impact any bit in
	1935	* the output digest, resulting in an unbiased distribution.
	1936	*
	1937	* @param h32 The hash to avalanche.
	1938	* @return The avalanched hash.
	1939	*/
	1940	static xxh_u32 XXH32_avalanche(xxh_u32 h32)
	1941	{
	1942	h32 ^= h32 >> 15;
	1943	h32 *= XXH_PRIME32_2;
	1944	h32 ^= h32 >> 13;
	1945	h32 *= XXH_PRIME32_3;
	1946	h32 ^= h32 >> 16;
	1947	return(h32);
	1948	}
	1949
	1950	#define XXH_get32bits(p) XXH_readLE32_align(p, align)
	1951
	1952	/*!
	1953	* @internal
	1954	* @brief Processes the last 0-15 bytes of @p ptr.
	1955	*
	1956	* There may be up to 15 bytes remaining to consume from the input.
	1957	* This final stage will digest them to ensure that all input bytes are present
	1958	* in the final mix.
	1959	*
	1960	* @param h32 The hash to finalize.
	1961	* @param ptr The pointer to the remaining input.
	1962	* @param len The remaining length, modulo 16.
	1963	* @param align Whether @p ptr is aligned.
	1964	* @return The finalized hash.
	1965	*/
	1966	static xxh_u32
	1967	XXH32_finalize(xxh_u32 h32, const xxh_u8* ptr, size_t len, XXH_alignment align)
	1968	{
	1969	#define XXH_PROCESS1 do { \
	1970	h32 += (ptr++) XXH_PRIME32_5; \
	1971	h32 = XXH_rotl32(h32, 11) * XXH_PRIME32_1; \
	1972	} while (0)
	1973
	1974	#define XXH_PROCESS4 do { \
	1975	h32 += XXH_get32bits(ptr) * XXH_PRIME32_3; \
	1976	ptr += 4; \
	1977	h32 = XXH_rotl32(h32, 17) * XXH_PRIME32_4; \
	1978	} while (0)
	1979
	1980	if (ptr==NULL) XXH_ASSERT(len == 0);
	1981
	1982	/* Compact rerolled version; generally faster */
	1983	if (!XXH32_ENDJMP) {
	1984	len &= 15;
	1985	while (len >= 4) {
	1986	XXH_PROCESS4;
	1987	len -= 4;
	1988	}
	1989	while (len > 0) {
	1990	XXH_PROCESS1;
	1991	--len;
	1992	}
	1993	return XXH32_avalanche(h32);
	1994	} else {
	1995	switch(len&15) /* or switch(bEnd - p) */ {
	1996	case 12: XXH_PROCESS4;
	1997	XXH_FALLTHROUGH;
	1998	case 8: XXH_PROCESS4;
	1999	XXH_FALLTHROUGH;
	2000	case 4: XXH_PROCESS4;
	2001	return XXH32_avalanche(h32);
	2002
	2003	case 13: XXH_PROCESS4;
	2004	XXH_FALLTHROUGH;
	2005	case 9: XXH_PROCESS4;
	2006	XXH_FALLTHROUGH;
	2007	case 5: XXH_PROCESS4;
	2008	XXH_PROCESS1;
	2009	return XXH32_avalanche(h32);
	2010
	2011	case 14: XXH_PROCESS4;
	2012	XXH_FALLTHROUGH;
	2013	case 10: XXH_PROCESS4;
	2014	XXH_FALLTHROUGH;
	2015	case 6: XXH_PROCESS4;
	2016	XXH_PROCESS1;
	2017	XXH_PROCESS1;
	2018	return XXH32_avalanche(h32);
	2019
	2020	case 15: XXH_PROCESS4;
	2021	XXH_FALLTHROUGH;
	2022	case 11: XXH_PROCESS4;
	2023	XXH_FALLTHROUGH;
	2024	case 7: XXH_PROCESS4;
	2025	XXH_FALLTHROUGH;
	2026	case 3: XXH_PROCESS1;
	2027	XXH_FALLTHROUGH;
	2028	case 2: XXH_PROCESS1;
	2029	XXH_FALLTHROUGH;
	2030	case 1: XXH_PROCESS1;
	2031	XXH_FALLTHROUGH;
	2032	case 0: return XXH32_avalanche(h32);
	2033	}
	2034	XXH_ASSERT(0);
	2035	return h32; /* reaching this point is deemed impossible */
	2036	}
	2037	}
	2038
	2039	#ifdef XXH_OLD_NAMES
	2040	# define PROCESS1 XXH_PROCESS1
	2041	# define PROCESS4 XXH_PROCESS4
	2042	#else
	2043	# undef XXH_PROCESS1
	2044	# undef XXH_PROCESS4
	2045	#endif
	2046
	2047	/*!
	2048	* @internal
	2049	* @brief The implementation for @ref XXH32().
	2050	*
	2051	* @param input , len , seed Directly passed from @ref XXH32().
	2052	* @param align Whether @p input is aligned.
	2053	* @return The calculated hash.
	2054	*/
	2055	XXH_FORCE_INLINE xxh_u32
	2056	XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align)
	2057	{
	2058	xxh_u32 h32;
	2059
	2060	if (input==NULL) XXH_ASSERT(len == 0);
	2061
	2062	if (len>=16) {
	2063	const xxh_u8* const bEnd = input + len;
	2064	const xxh_u8* const limit = bEnd - 15;
	2065	xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
	2066	xxh_u32 v2 = seed + XXH_PRIME32_2;
	2067	xxh_u32 v3 = seed + 0;
	2068	xxh_u32 v4 = seed - XXH_PRIME32_1;
	2069
	2070	do {
	2071	v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4;
	2072	v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4;
	2073	v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4;
	2074	v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4;
	2075	} while (input < limit);
	2076
	2077	h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7)
	2078	+ XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
	2079	} else {
	2080	h32 = seed + XXH_PRIME32_5;
	2081	}
	2082
	2083	h32 += (xxh_u32)len;
	2084
	2085	return XXH32_finalize(h32, input, len&15, align);
	2086	}
	2087
	2088	/! @ingroup xxh32_family /
	2089	XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed)
	2090	{
	2091	#if 0
	2092	/* Simple version, good for code maintenance, but unfortunately slow for small inputs */
	2093	XXH32_state_t state;
	2094	XXH32_reset(&state, seed);
	2095	XXH32_update(&state, (const xxh_u8*)input, len);
	2096	return XXH32_digest(&state);
	2097	#else
	2098	if (XXH_FORCE_ALIGN_CHECK) {
	2099	if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */
	2100	return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned);
	2101	} }
	2102
	2103	return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned);
	2104	#endif
	2105	}
	2106
	2107
	2108
	2109	/***** Hash streaming *****/
	2110	/*!
	2111	* @ingroup xxh32_family
	2112	*/
	2113	XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
	2114	{
	2115	return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
	2116	}
	2117	/! @ingroup xxh32_family /
	2118	XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
	2119	{
	2120	XXH_free(statePtr);
	2121	return XXH_OK;
	2122	}
	2123
	2124	/! @ingroup xxh32_family /
	2125	XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState)
	2126	{
	2127	XXH_memcpy(dstState, srcState, sizeof(*dstState));
	2128	}
	2129
	2130	/! @ingroup xxh32_family /
	2131	XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed)
	2132	{
	2133	XXH_ASSERT(statePtr != NULL);
	2134	memset(statePtr, 0, sizeof(*statePtr));
	2135	statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
	2136	statePtr->v[1] = seed + XXH_PRIME32_2;
	2137	statePtr->v[2] = seed + 0;
	2138	statePtr->v[3] = seed - XXH_PRIME32_1;
	2139	return XXH_OK;
	2140	}
	2141
	2142
	2143	/! @ingroup xxh32_family /
	2144	XXH_PUBLIC_API XXH_errorcode
	2145	XXH32_update(XXH32_state_t* state, const void* input, size_t len)
	2146	{
	2147	if (input==NULL) {
	2148	XXH_ASSERT(len == 0);
	2149	return XXH_OK;
	2150	}
	2151
	2152	{ const xxh_u8* p = (const xxh_u8*)input;
	2153	const xxh_u8* const bEnd = p + len;
	2154
	2155	state->total_len_32 += (XXH32_hash_t)len;
	2156	state->large_len \|= (XXH32_hash_t)((len>=16) \| (state->total_len_32>=16));
	2157
	2158	if (state->memsize + len < 16) { /* fill in tmp buffer */
	2159	XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len);
	2160	state->memsize += (XXH32_hash_t)len;
	2161	return XXH_OK;
	2162	}
	2163
	2164	if (state->memsize) { /* some data left from previous update */
	2165	XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize);
	2166	{ const xxh_u32* p32 = state->mem32;
	2167	state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++;
	2168	state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++;
	2169	state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++;
	2170	state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32));
	2171	}
	2172	p += 16-state->memsize;
	2173	state->memsize = 0;
	2174	}
	2175
	2176	if (p <= bEnd-16) {
	2177	const xxh_u8* const limit = bEnd - 16;
	2178
	2179	do {
	2180	state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4;
	2181	state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4;
	2182	state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4;
	2183	state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4;
	2184	} while (p<=limit);
	2185
	2186	}
	2187
	2188	if (p < bEnd) {
	2189	XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
	2190	state->memsize = (unsigned)(bEnd-p);
	2191	}
	2192	}
	2193
	2194	return XXH_OK;
	2195	}
	2196
	2197
	2198	/! @ingroup xxh32_family /
	2199	XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state)
	2200	{
	2201	xxh_u32 h32;
	2202
	2203	if (state->large_len) {
	2204	h32 = XXH_rotl32(state->v[0], 1)
	2205	+ XXH_rotl32(state->v[1], 7)
	2206	+ XXH_rotl32(state->v[2], 12)
	2207	+ XXH_rotl32(state->v[3], 18);
	2208	} else {
	2209	h32 = state->v[2] /* == seed */ + XXH_PRIME32_5;
	2210	}
	2211
	2212	h32 += state->total_len_32;
	2213
	2214	return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned);
	2215	}
	2216
	2217
	2218	/***** Canonical representation *****/
	2219
	2220	/*!
	2221	* @ingroup xxh32_family
	2222	* The default return values from XXH functions are unsigned 32 and 64 bit
	2223	* integers.
	2224	*
	2225	* The canonical representation uses big endian convention, the same convention
	2226	* as human-readable numbers (large digits first).
	2227	*
	2228	* This way, hash values can be written into a file or buffer, remaining
	2229	* comparable across different systems.
	2230	*
	2231	* The following functions allow transformation of hash values to and from their
	2232	* canonical format.
	2233	*/
	2234	XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
	2235	{
	2236	/* XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); */
	2237	if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
	2238	XXH_memcpy(dst, &hash, sizeof(*dst));
	2239	}
	2240	/! @ingroup xxh32_family /
	2241	XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
	2242	{
	2243	return XXH_readBE32(src);
	2244	}
	2245
	2246
	2247	#ifndef XXH_NO_LONG_LONG
	2248
	2249	/* *******************************************************************
	2250	* 64-bit hash functions
	2251	*********************************************************************/
	2252	/*!
	2253	* @}
	2254	* @ingroup impl
	2255	* @{
	2256	*/
	2257	/***** Memory access *****/
	2258
	2259	typedef XXH64_hash_t xxh_u64;
	2260
	2261	#ifdef XXH_OLD_NAMES
	2262	# define U64 xxh_u64
	2263	#endif
	2264
	2265	#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
	2266	/*
	2267	* Manual byteshift. Best for old compilers which don't inline memcpy.
	2268	* We actually directly use XXH_readLE64 and XXH_readBE64.
	2269	*/
	2270	#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
	2271
	2272	/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
	2273	static xxh_u64 XXH_read64(const void* memPtr)
	2274	{
	2275	return (const xxh_u64) memPtr;
	2276	}
	2277
	2278	#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
	2279
	2280	/*
	2281	* __pack instructions are safer, but compiler specific, hence potentially
	2282	* problematic for some compilers.
	2283	*
	2284	* Currently only defined for GCC and ICC.
	2285	*/
	2286	#ifdef XXH_OLD_NAMES
	2287	typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64;
	2288	#endif
	2289	static xxh_u64 XXH_read64(const void* ptr)
	2290	{
	2291	typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) xxh_unalign64;
	2292	return ((const xxh_unalign64*)ptr)->u64;
	2293	}
	2294
	2295	#else
	2296
	2297	/*
	2298	* Portable and safe solution. Generally efficient.
	2299	* see: http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
	2300	*/
	2301	static xxh_u64 XXH_read64(const void* memPtr)
	2302	{
	2303	xxh_u64 val;
	2304	XXH_memcpy(&val, memPtr, sizeof(val));
	2305	return val;
	2306	}
	2307
	2308	#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
	2309
	2310	#if defined(_MSC_VER) /* Visual Studio */
	2311	# define XXH_swap64 _byteswap_uint64
	2312	#elif XXH_GCC_VERSION >= 403
	2313	# define XXH_swap64 __builtin_bswap64
	2314	#else
	2315	static xxh_u64 XXH_swap64(xxh_u64 x)
	2316	{
	2317	return ((x << 56) & 0xff00000000000000ULL) \|
	2318	((x << 40) & 0x00ff000000000000ULL) \|
	2319	((x << 24) & 0x0000ff0000000000ULL) \|
	2320	((x << 8) & 0x000000ff00000000ULL) \|
	2321	((x >> 8) & 0x00000000ff000000ULL) \|
	2322	((x >> 24) & 0x0000000000ff0000ULL) \|
	2323	((x >> 40) & 0x000000000000ff00ULL) \|
	2324	((x >> 56) & 0x00000000000000ffULL);
	2325	}
	2326	#endif
	2327
	2328
	2329	/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */
	2330	#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
	2331
	2332	XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr)
	2333	{
	2334	const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
	2335	return bytePtr[0]
	2336	\| ((xxh_u64)bytePtr[1] << 8)
	2337	\| ((xxh_u64)bytePtr[2] << 16)
	2338	\| ((xxh_u64)bytePtr[3] << 24)
	2339	\| ((xxh_u64)bytePtr[4] << 32)
	2340	\| ((xxh_u64)bytePtr[5] << 40)
	2341	\| ((xxh_u64)bytePtr[6] << 48)
	2342	\| ((xxh_u64)bytePtr[7] << 56);
	2343	}
	2344
	2345	XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr)
	2346	{
	2347	const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
	2348	return bytePtr[7]
	2349	\| ((xxh_u64)bytePtr[6] << 8)
	2350	\| ((xxh_u64)bytePtr[5] << 16)
	2351	\| ((xxh_u64)bytePtr[4] << 24)
	2352	\| ((xxh_u64)bytePtr[3] << 32)
	2353	\| ((xxh_u64)bytePtr[2] << 40)
	2354	\| ((xxh_u64)bytePtr[1] << 48)
	2355	\| ((xxh_u64)bytePtr[0] << 56);
	2356	}
	2357
	2358	#else
	2359	XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr)
	2360	{
	2361	return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
	2362	}
	2363
	2364	static xxh_u64 XXH_readBE64(const void* ptr)
	2365	{
	2366	return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
	2367	}
	2368	#endif
	2369
	2370	XXH_FORCE_INLINE xxh_u64
	2371	XXH_readLE64_align(const void* ptr, XXH_alignment align)
	2372	{
	2373	if (align==XXH_unaligned)
	2374	return XXH_readLE64(ptr);
	2375	else
	2376	return XXH_CPU_LITTLE_ENDIAN ? (const xxh_u64)ptr : XXH_swap64((const xxh_u64)ptr);
	2377	}
	2378
	2379
	2380	/***** xxh64 *****/
	2381	/*!
	2382	* @}
	2383	* @defgroup xxh64_impl XXH64 implementation
	2384	* @ingroup impl
	2385	* @{
	2386	*/
	2387	/* #define rather that static const, to be used as initializers */
	2388	#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /!< 0b1001111000110111011110011011000110000101111010111100101010000111 /
	2389	#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /!< 0b1100001010110010101011100011110100100111110101001110101101001111 /
	2390	#define XXH_PRIME64_3 0x165667B19E3779F9ULL /!< 0b0001011001010110011001111011000110011110001101110111100111111001 /
	2391	#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /!< 0b1000010111101011110010100111011111000010101100101010111001100011 /
	2392	#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /!< 0b0010011111010100111010110010111100010110010101100110011111000101 /
	2393
	2394	#ifdef XXH_OLD_NAMES
	2395	# define PRIME64_1 XXH_PRIME64_1
	2396	# define PRIME64_2 XXH_PRIME64_2
	2397	# define PRIME64_3 XXH_PRIME64_3
	2398	# define PRIME64_4 XXH_PRIME64_4
	2399	# define PRIME64_5 XXH_PRIME64_5
	2400	#endif
	2401
	2402	static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input)
	2403	{
	2404	acc += input * XXH_PRIME64_2;
	2405	acc = XXH_rotl64(acc, 31);
	2406	acc *= XXH_PRIME64_1;
	2407	return acc;
	2408	}
	2409
	2410	static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val)
	2411	{
	2412	val = XXH64_round(0, val);
	2413	acc ^= val;
	2414	acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4;
	2415	return acc;
	2416	}
	2417
	2418	static xxh_u64 XXH64_avalanche(xxh_u64 h64)
	2419	{
	2420	h64 ^= h64 >> 33;
	2421	h64 *= XXH_PRIME64_2;
	2422	h64 ^= h64 >> 29;
	2423	h64 *= XXH_PRIME64_3;
	2424	h64 ^= h64 >> 32;
	2425	return h64;
	2426	}
	2427
	2428
	2429	#define XXH_get64bits(p) XXH_readLE64_align(p, align)
	2430
	2431	static xxh_u64
	2432	XXH64_finalize(xxh_u64 h64, const xxh_u8* ptr, size_t len, XXH_alignment align)
	2433	{
	2434	if (ptr==NULL) XXH_ASSERT(len == 0);
	2435	len &= 31;
	2436	while (len >= 8) {
	2437	xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr));
	2438	ptr += 8;
	2439	h64 ^= k1;
	2440	h64 = XXH_rotl64(h64,27) * XXH_PRIME64_1 + XXH_PRIME64_4;
	2441	len -= 8;
	2442	}
	2443	if (len >= 4) {
	2444	h64 ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1;
	2445	ptr += 4;
	2446	h64 = XXH_rotl64(h64, 23) * XXH_PRIME64_2 + XXH_PRIME64_3;
	2447	len -= 4;
	2448	}
	2449	while (len > 0) {
	2450	h64 ^= (ptr++) XXH_PRIME64_5;
	2451	h64 = XXH_rotl64(h64, 11) * XXH_PRIME64_1;
	2452	--len;
	2453	}
	2454	return XXH64_avalanche(h64);
	2455	}
	2456
	2457	#ifdef XXH_OLD_NAMES
	2458	# define PROCESS1_64 XXH_PROCESS1_64
	2459	# define PROCESS4_64 XXH_PROCESS4_64
	2460	# define PROCESS8_64 XXH_PROCESS8_64
	2461	#else
	2462	# undef XXH_PROCESS1_64
	2463	# undef XXH_PROCESS4_64
	2464	# undef XXH_PROCESS8_64
	2465	#endif
	2466
	2467	XXH_FORCE_INLINE xxh_u64
	2468	XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align)
	2469	{
	2470	xxh_u64 h64;
	2471	if (input==NULL) XXH_ASSERT(len == 0);
	2472
	2473	if (len>=32) {
	2474	const xxh_u8* const bEnd = input + len;
	2475	const xxh_u8* const limit = bEnd - 31;
	2476	xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
	2477	xxh_u64 v2 = seed + XXH_PRIME64_2;
	2478	xxh_u64 v3 = seed + 0;
	2479	xxh_u64 v4 = seed - XXH_PRIME64_1;
	2480
	2481	do {
	2482	v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8;
	2483	v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8;
	2484	v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8;
	2485	v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8;
	2486	} while (input<limit);
	2487
	2488	h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
	2489	h64 = XXH64_mergeRound(h64, v1);
	2490	h64 = XXH64_mergeRound(h64, v2);
	2491	h64 = XXH64_mergeRound(h64, v3);
	2492	h64 = XXH64_mergeRound(h64, v4);
	2493
	2494	} else {
	2495	h64 = seed + XXH_PRIME64_5;
	2496	}
	2497
	2498	h64 += (xxh_u64) len;
	2499
	2500	return XXH64_finalize(h64, input, len, align);
	2501	}
	2502
	2503
	2504	/! @ingroup xxh64_family /
	2505	XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t len, XXH64_hash_t seed)
	2506	{
	2507	#if 0
	2508	/* Simple version, good for code maintenance, but unfortunately slow for small inputs */
	2509	XXH64_state_t state;
	2510	XXH64_reset(&state, seed);
	2511	XXH64_update(&state, (const xxh_u8*)input, len);
	2512	return XXH64_digest(&state);
	2513	#else
	2514	if (XXH_FORCE_ALIGN_CHECK) {
	2515	if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */
	2516	return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned);
	2517	} }
	2518
	2519	return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned);
	2520
	2521	#endif
	2522	}
	2523
	2524	/***** Hash Streaming *****/
	2525
	2526	/! @ingroup xxh64_family/
	2527	XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
	2528	{
	2529	return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
	2530	}
	2531	/! @ingroup xxh64_family /
	2532	XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
	2533	{
	2534	XXH_free(statePtr);
	2535	return XXH_OK;
	2536	}
	2537
	2538	/! @ingroup xxh64_family /
	2539	XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState)
	2540	{
	2541	XXH_memcpy(dstState, srcState, sizeof(*dstState));
	2542	}
	2543
	2544	/! @ingroup xxh64_family /
	2545	XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, XXH64_hash_t seed)
	2546	{
	2547	XXH_ASSERT(statePtr != NULL);
	2548	memset(statePtr, 0, sizeof(*statePtr));
	2549	statePtr->v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
	2550	statePtr->v[1] = seed + XXH_PRIME64_2;
	2551	statePtr->v[2] = seed + 0;
	2552	statePtr->v[3] = seed - XXH_PRIME64_1;
	2553	return XXH_OK;
	2554	}
	2555
	2556	/! @ingroup xxh64_family /
	2557	XXH_PUBLIC_API XXH_errorcode
	2558	XXH64_update (XXH64_state_t* state, const void* input, size_t len)
	2559	{
	2560	if (input==NULL) {
	2561	XXH_ASSERT(len == 0);
	2562	return XXH_OK;
	2563	}
	2564
	2565	{ const xxh_u8* p = (const xxh_u8*)input;
	2566	const xxh_u8* const bEnd = p + len;
	2567
	2568	state->total_len += len;
	2569
	2570	if (state->memsize + len < 32) { /* fill in tmp buffer */
	2571	XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len);
	2572	state->memsize += (xxh_u32)len;
	2573	return XXH_OK;
	2574	}
	2575
	2576	if (state->memsize) { /* tmp buffer is full */
	2577	XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize);
	2578	state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0));
	2579	state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1));
	2580	state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2));
	2581	state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3));
	2582	p += 32 - state->memsize;
	2583	state->memsize = 0;
	2584	}
	2585
	2586	if (p+32 <= bEnd) {
	2587	const xxh_u8* const limit = bEnd - 32;
	2588
	2589	do {
	2590	state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8;
	2591	state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8;
	2592	state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8;
	2593	state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8;
	2594	} while (p<=limit);
	2595
	2596	}
	2597
	2598	if (p < bEnd) {
	2599	XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
	2600	state->memsize = (unsigned)(bEnd-p);
	2601	}
	2602	}
	2603
	2604	return XXH_OK;
	2605	}
	2606
	2607
	2608	/! @ingroup xxh64_family /
	2609	XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t* state)
	2610	{
	2611	xxh_u64 h64;
	2612
	2613	if (state->total_len >= 32) {
	2614	h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18);
	2615	h64 = XXH64_mergeRound(h64, state->v[0]);
	2616	h64 = XXH64_mergeRound(h64, state->v[1]);
	2617	h64 = XXH64_mergeRound(h64, state->v[2]);
	2618	h64 = XXH64_mergeRound(h64, state->v[3]);
	2619	} else {
	2620	h64 = state->v[2] /seed/ + XXH_PRIME64_5;
	2621	}
	2622
	2623	h64 += (xxh_u64) state->total_len;
	2624
	2625	return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned);
	2626	}
	2627
	2628
	2629	/***** Canonical representation *****/
	2630
	2631	/! @ingroup xxh64_family /
	2632	XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash)
	2633	{
	2634	/* XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); */
	2635	if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
	2636	XXH_memcpy(dst, &hash, sizeof(*dst));
	2637	}
	2638
	2639	/! @ingroup xxh64_family /
	2640	XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)
	2641	{
	2642	return XXH_readBE64(src);
	2643	}
	2644
	2645	#ifndef XXH_NO_XXH3
	2646
	2647	/* *********************************************************************
	2648	* XXH3
	2649	* New generation hash designed for speed on small keys and vectorization
	2650	************************************************************************ */
	2651	/*!
	2652	* @}
	2653	* @defgroup xxh3_impl XXH3 implementation
	2654	* @ingroup impl
	2655	* @{
	2656	*/
	2657
	2658	/* === Compiler specifics === */
	2659
	2660	#if ((defined(sun) \|\| defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */
	2661	# define XXH_RESTRICT /* disable */
	2662	#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */
	2663	# define XXH_RESTRICT restrict
	2664	#else
	2665	/* Note: it might be useful to define __restrict or __restrict__ for some C++ compilers */
	2666	# define XXH_RESTRICT /* disable */
	2667	#endif
	2668
	2669	#if (defined(__GNUC__) && (__GNUC__ >= 3)) \
	2670	\|\| (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \
	2671	\|\| defined(__clang__)
	2672	# define XXH_likely(x) __builtin_expect(x, 1)
	2673	# define XXH_unlikely(x) __builtin_expect(x, 0)
	2674	#else
	2675	# define XXH_likely(x) (x)
	2676	# define XXH_unlikely(x) (x)
	2677	#endif
	2678
	2679	#if defined(__GNUC__) \|\| defined(__clang__)
	2680	# if defined(__ARM_NEON__) \|\| defined(__ARM_NEON) \
	2681	\|\| defined(__aarch64__) \|\| defined(_M_ARM) \
	2682	\|\| defined(_M_ARM64) \|\| defined(_M_ARM64EC)
	2683	# define inline __inline__ /* circumvent a clang bug */
	2684	# include <arm_neon.h>
	2685	# undef inline
	2686	# elif defined(__AVX2__)
	2687	# include <immintrin.h>
	2688	# elif defined(__SSE2__)
	2689	# include <emmintrin.h>
	2690	# endif
	2691	#endif
	2692
	2693	#if defined(_MSC_VER)
	2694	# include <intrin.h>
	2695	#endif
	2696
	2697	/*
	2698	* One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while
	2699	* remaining a true 64-bit/128-bit hash function.
	2700	*
	2701	* This is done by prioritizing a subset of 64-bit operations that can be
	2702	* emulated without too many steps on the average 32-bit machine.
	2703	*
	2704	* For example, these two lines seem similar, and run equally fast on 64-bit:
	2705	*
	2706	* xxh_u64 x;
	2707	* x ^= (x >> 47); // good
	2708	* x ^= (x >> 13); // bad
	2709	*
	2710	* However, to a 32-bit machine, there is a major difference.
	2711	*
	2712	* x ^= (x >> 47) looks like this:
	2713	*
	2714	* x.lo ^= (x.hi >> (47 - 32));
	2715	*
	2716	* while x ^= (x >> 13) looks like this:
	2717	*
	2718	* // note: funnel shifts are not usually cheap.
	2719	* x.lo ^= (x.lo >> 13) \| (x.hi << (32 - 13));
	2720	* x.hi ^= (x.hi >> 13);
	2721	*
	2722	* The first one is significantly faster than the second, simply because the
	2723	* shift is larger than 32. This means:
	2724	* - All the bits we need are in the upper 32 bits, so we can ignore the lower
	2725	* 32 bits in the shift.
	2726	* - The shift result will always fit in the lower 32 bits, and therefore,
	2727	* we can ignore the upper 32 bits in the xor.
	2728	*
	2729	* Thanks to this optimization, XXH3 only requires these features to be efficient:
	2730	*
	2731	* - Usable unaligned access
	2732	* - A 32-bit or 64-bit ALU
	2733	* - If 32-bit, a decent ADC instruction
	2734	* - A 32 or 64-bit multiply with a 64-bit result
	2735	* - For the 128-bit variant, a decent byteswap helps short inputs.
	2736	*
	2737	* The first two are already required by XXH32, and almost all 32-bit and 64-bit
	2738	* platforms which can run XXH32 can run XXH3 efficiently.
	2739	*
	2740	* Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one
	2741	* notable exception.
	2742	*
	2743	* First of all, Thumb-1 lacks support for the UMULL instruction which
	2744	* performs the important long multiply. This means numerous __aeabi_lmul
	2745	* calls.
	2746	*
	2747	* Second of all, the 8 functional registers are just not enough.
	2748	* Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need
	2749	* Lo registers, and this shuffling results in thousands more MOVs than A32.
	2750	*
	2751	* A32 and T32 don't have this limitation. They can access all 14 registers,
	2752	* do a 32->64 multiply with UMULL, and the flexible operand allowing free
	2753	* shifts is helpful, too.
	2754	*
	2755	* Therefore, we do a quick sanity check.
	2756	*
	2757	* If compiling Thumb-1 for a target which supports ARM instructions, we will
	2758	* emit a warning, as it is not a "sane" platform to compile for.
	2759	*
	2760	* Usually, if this happens, it is because of an accident and you probably need
	2761	* to specify -march, as you likely meant to compile for a newer architecture.
	2762	*
	2763	* Credit: large sections of the vectorial and asm source code paths
	2764	* have been contributed by @easyaspi314
	2765	*/
	2766	#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM)
	2767	# warning "XXH3 is highly inefficient without ARM or Thumb-2."
	2768	#endif
	2769
	2770	/* ==========================================
	2771	* Vectorization detection
	2772	* ========================================== */
	2773
	2774	#ifdef XXH_DOXYGEN
	2775	/*!
	2776	* @ingroup tuning
	2777	* @brief Overrides the vectorization implementation chosen for XXH3.
	2778	*
	2779	* Can be defined to 0 to disable SIMD or any of the values mentioned in
	2780	* @ref XXH_VECTOR_TYPE.
	2781	*
	2782	* If this is not defined, it uses predefined macros to determine the best
	2783	* implementation.
	2784	*/
	2785	# define XXH_VECTOR XXH_SCALAR
	2786	/*!
	2787	* @ingroup tuning
	2788	* @brief Possible values for @ref XXH_VECTOR.
	2789	*
	2790	* Note that these are actually implemented as macros.
	2791	*
	2792	* If this is not defined, it is detected automatically.
	2793	* @ref XXH_X86DISPATCH overrides this.
	2794	*/
	2795	enum XXH_VECTOR_TYPE /* fake enum */ {
	2796	XXH_SCALAR = 0, /!< Portable scalar version /
	2797	XXH_SSE2 = 1, /*!<
	2798	* SSE2 for Pentium 4, Opteron, all x86_64.
	2799	*
	2800	* @note SSE2 is also guaranteed on Windows 10, macOS, and
	2801	* Android x86.
	2802	*/
	2803	XXH_AVX2 = 2, /!< AVX2 for Haswell and Bulldozer /
	2804	XXH_AVX512 = 3, /!< AVX512 for Skylake and Icelake /
	2805	XXH_NEON = 4, /!< NEON for most ARMv7-A and all AArch64 /
	2806	XXH_VSX = 5, /!< VSX and ZVector for POWER8/z13 (64-bit) /
	2807	};
	2808	/*!
	2809	* @ingroup tuning
	2810	* @brief Selects the minimum alignment for XXH3's accumulators.
	2811	*
	2812	* When using SIMD, this should match the alignment reqired for said vector
	2813	* type, so, for example, 32 for AVX2.
	2814	*
	2815	* Default: Auto detected.
	2816	*/
	2817	# define XXH_ACC_ALIGN 8
	2818	#endif
	2819
	2820	/* Actual definition */
	2821	#ifndef XXH_DOXYGEN
	2822	# define XXH_SCALAR 0
	2823	# define XXH_SSE2 1
	2824	# define XXH_AVX2 2
	2825	# define XXH_AVX512 3
	2826	# define XXH_NEON 4
	2827	# define XXH_VSX 5
	2828	#endif
	2829
	2830	#ifndef XXH_VECTOR /* can be defined on command line */
	2831	# if ( \
	2832	defined(__ARM_NEON__) \|\| defined(__ARM_NEON) /* gcc */ \
	2833	\|\| defined(_M_ARM) \|\| defined(_M_ARM64) \|\| defined(_M_ARM64EC) /* msvc */ \
	2834	) && ( \
	2835	defined(_WIN32) \|\| defined(__LITTLE_ENDIAN__) /* little endian only */ \
	2836	\|\| (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \
	2837	)
	2838	# define XXH_VECTOR XXH_NEON
	2839	# elif defined(__AVX512F__)
	2840	# define XXH_VECTOR XXH_AVX512
	2841	# elif defined(__AVX2__)
	2842	# define XXH_VECTOR XXH_AVX2
	2843	# elif defined(__SSE2__) \|\| defined(_M_AMD64) \|\| defined(_M_X64) \|\| (defined(_M_IX86_FP) && (_M_IX86_FP == 2))
	2844	# define XXH_VECTOR XXH_SSE2
	2845	# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \
	2846	\|\| (defined(__s390x__) && defined(__VEC__)) \
	2847	&& defined(__GNUC__) /* TODO: IBM XL */
	2848	# define XXH_VECTOR XXH_VSX
	2849	# else
	2850	# define XXH_VECTOR XXH_SCALAR
	2851	# endif
	2852	#endif
	2853
	2854	/*
	2855	* Controls the alignment of the accumulator,
	2856	* for compatibility with aligned vector loads, which are usually faster.
	2857	*/
	2858	#ifndef XXH_ACC_ALIGN
	2859	# if defined(XXH_X86DISPATCH)
	2860	# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */
	2861	# elif XXH_VECTOR == XXH_SCALAR /* scalar */
	2862	# define XXH_ACC_ALIGN 8
	2863	# elif XXH_VECTOR == XXH_SSE2 /* sse2 */
	2864	# define XXH_ACC_ALIGN 16
	2865	# elif XXH_VECTOR == XXH_AVX2 /* avx2 */
	2866	# define XXH_ACC_ALIGN 32
	2867	# elif XXH_VECTOR == XXH_NEON /* neon */
	2868	# define XXH_ACC_ALIGN 16
	2869	# elif XXH_VECTOR == XXH_VSX /* vsx */
	2870	# define XXH_ACC_ALIGN 16
	2871	# elif XXH_VECTOR == XXH_AVX512 /* avx512 */
	2872	# define XXH_ACC_ALIGN 64
	2873	# endif
	2874	#endif
	2875
	2876	#if defined(XXH_X86DISPATCH) \|\| XXH_VECTOR == XXH_SSE2 \
	2877	\|\| XXH_VECTOR == XXH_AVX2 \|\| XXH_VECTOR == XXH_AVX512
	2878	# define XXH_SEC_ALIGN XXH_ACC_ALIGN
	2879	#else
	2880	# define XXH_SEC_ALIGN 8
	2881	#endif
	2882
	2883	/*
	2884	* UGLY HACK:
	2885	* GCC usually generates the best code with -O3 for xxHash.
	2886	*
	2887	* However, when targeting AVX2, it is overzealous in its unrolling resulting
	2888	* in code roughly 3/4 the speed of Clang.
	2889	*
	2890	* There are other issues, such as GCC splitting _mm256_loadu_si256 into
	2891	* _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which
	2892	* only applies to Sandy and Ivy Bridge... which don't even support AVX2.
	2893	*
	2894	* That is why when compiling the AVX2 version, it is recommended to use either
	2895	* -O2 -mavx2 -march=haswell
	2896	* or
	2897	* -O2 -mavx2 -mno-avx256-split-unaligned-load
	2898	* for decent performance, or to use Clang instead.
	2899	*
	2900	* Fortunately, we can control the first one with a pragma that forces GCC into
	2901	* -O2, but the other one we can't control without "failed to inline always
	2902	* inline function due to target mismatch" warnings.
	2903	*/
	2904	#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \
	2905	&& defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
	2906	&& defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */
	2907	# pragma GCC push_options
	2908	# pragma GCC optimize("-O2")
	2909	#endif
	2910
	2911
	2912	#if XXH_VECTOR == XXH_NEON
	2913	/*
	2914	* NEON's setup for vmlal_u32 is a little more complicated than it is on
	2915	* SSE2, AVX2, and VSX.
	2916	*
	2917	* While PMULUDQ and VMULEUW both perform a mask, VMLAL.U32 performs an upcast.
	2918	*
	2919	* To do the same operation, the 128-bit 'Q' register needs to be split into
	2920	* two 64-bit 'D' registers, performing this operation::
	2921	*
	2922	* [ a \| b ]
	2923	* \| '---------. .--------' \|
	2924	* \| x \|
	2925	* \| .---------' '--------. \|
	2926	* [ a & 0xFFFFFFFF \| b & 0xFFFFFFFF ],[ a >> 32 \| b >> 32 ]
	2927	*
	2928	* Due to significant changes in aarch64, the fastest method for aarch64 is
	2929	* completely different than the fastest method for ARMv7-A.
	2930	*
	2931	* ARMv7-A treats D registers as unions overlaying Q registers, so modifying
	2932	* D11 will modify the high half of Q5. This is similar to how modifying AH
	2933	* will only affect bits 8-15 of AX on x86.
	2934	*
	2935	* VZIP takes two registers, and puts even lanes in one register and odd lanes
	2936	* in the other.
	2937	*
	2938	* On ARMv7-A, this strangely modifies both parameters in place instead of
	2939	* taking the usual 3-operand form.
	2940	*
	2941	* Therefore, if we want to do this, we can simply use a D-form VZIP.32 on the
	2942	* lower and upper halves of the Q register to end up with the high and low
	2943	* halves where we want - all in one instruction.
	2944	*
	2945	* vzip.32 d10, d11 @ d10 = { d10[0], d11[0] }; d11 = { d10[1], d11[1] }
	2946	*
	2947	* Unfortunately we need inline assembly for this: Instructions modifying two
	2948	* registers at once is not possible in GCC or Clang's IR, and they have to
	2949	* create a copy.
	2950	*
	2951	* aarch64 requires a different approach.
	2952	*
	2953	* In order to make it easier to write a decent compiler for aarch64, many
	2954	* quirks were removed, such as conditional execution.
	2955	*
	2956	* NEON was also affected by this.
	2957	*
	2958	* aarch64 cannot access the high bits of a Q-form register, and writes to a
	2959	* D-form register zero the high bits, similar to how writes to W-form scalar
	2960	* registers (or DWORD registers on x86_64) work.
	2961	*
	2962	* The formerly free vget_high intrinsics now require a vext (with a few
	2963	* exceptions)
	2964	*
	2965	* Additionally, VZIP was replaced by ZIP1 and ZIP2, which are the equivalent
	2966	* of PUNPCKL* and PUNPCKH* in SSE, respectively, in order to only modify one
	2967	* operand.
	2968	*
	2969	* The equivalent of the VZIP.32 on the lower and upper halves would be this
	2970	* mess:
	2971	*
	2972	* ext v2.4s, v0.4s, v0.4s, #2 // v2 = { v0[2], v0[3], v0[0], v0[1] }
	2973	* zip1 v1.2s, v0.2s, v2.2s // v1 = { v0[0], v2[0] }
	2974	* zip2 v0.2s, v0.2s, v1.2s // v0 = { v0[1], v2[1] }
	2975	*
	2976	* Instead, we use a literal downcast, vmovn_u64 (XTN), and vshrn_n_u64 (SHRN):
	2977	*
	2978	* shrn v1.2s, v0.2d, #32 // v1 = (uint32x2_t)(v0 >> 32);
	2979	* xtn v0.2s, v0.2d // v0 = (uint32x2_t)(v0 & 0xFFFFFFFF);
	2980	*
	2981	* This is available on ARMv7-A, but is less efficient than a single VZIP.32.
	2982	*/
	2983
	2984	/*!
	2985	* Function-like macro:
	2986	* void XXH_SPLIT_IN_PLACE(uint64x2_t &in, uint32x2_t &outLo, uint32x2_t &outHi)
	2987	* {
	2988	* outLo = (uint32x2_t)(in & 0xFFFFFFFF);
	2989	* outHi = (uint32x2_t)(in >> 32);
	2990	* in = UNDEFINED;
	2991	* }
	2992	*/
	2993	# if !defined(XXH_NO_VZIP_HACK) /* define to disable */ \
	2994	&& (defined(__GNUC__) \|\| defined(__clang__)) \
	2995	&& (defined(__arm__) \|\| defined(__thumb__) \|\| defined(_M_ARM))
	2996	# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \
	2997	do { \
	2998	/* Undocumented GCC/Clang operand modifier: %e0 = lower D half, %f0 = upper D half */ \
	2999	/* https://github.com/gcc-mirror/gcc/blob/38cf91e5/gcc/config/arm/arm.c#L22486 */ \
	3000	/* https://github.com/llvm-mirror/llvm/blob/2c4ca683/lib/Target/ARM/ARMAsmPrinter.cpp#L399 */ \
	3001	__asm__("vzip.32 %e0, %f0" : "+w" (in)); \
	3002	(outLo) = vget_low_u32 (vreinterpretq_u32_u64(in)); \
	3003	(outHi) = vget_high_u32(vreinterpretq_u32_u64(in)); \
	3004	} while (0)
	3005	# else
	3006	# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \
	3007	do { \
	3008	(outLo) = vmovn_u64 (in); \
	3009	(outHi) = vshrn_n_u64 ((in), 32); \
	3010	} while (0)
	3011	# endif
	3012
	3013	/*!
	3014	* @ingroup tuning
	3015	* @brief Controls the NEON to scalar ratio for XXH3
	3016	*
	3017	* On AArch64 when not optimizing for size, XXH3 will run 6 lanes using NEON and
	3018	* 2 lanes on scalar by default.
	3019	*
	3020	* This can be set to 2, 4, 6, or 8. ARMv7 will default to all 8 NEON lanes, as the
	3021	* emulated 64-bit arithmetic is too slow.
	3022	*
	3023	* Modern ARM CPUs are _very_ sensitive to how their pipelines are used.
	3024	*
	3025	* For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but it can't
	3026	* have more than 2 NEON (F0/F1) micro-ops. If you are only using NEON instructions,
	3027	* you are only using 2/3 of the CPU bandwidth.
	3028	*
	3029	* This is even more noticable on the more advanced cores like the A76 which
	3030	* can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once.
	3031	*
	3032	* Therefore, @ref XXH3_NEON_LANES lanes will be processed using NEON, and the
	3033	* remaining lanes will use scalar instructions. This improves the bandwidth
	3034	* and also gives the integer pipelines something to do besides twiddling loop
	3035	* counters and pointers.
	3036	*
	3037	* This change benefits CPUs with large micro-op buffers without negatively affecting
	3038	* other CPUs:
	3039	*
	3040	* \| Chipset \| Dispatch type \| NEON only \| 6:2 hybrid \| Diff. \|
	3041	* \|:----------------------\|:--------------------\|----------:\|-----------:\|------:\|
	3042	* \| Snapdragon 730 (A76) \| 2 NEON/8 micro-ops \| 8.8 GB/s \| 10.1 GB/s \| ~16% \|
	3043	* \| Snapdragon 835 (A73) \| 2 NEON/3 micro-ops \| 5.1 GB/s \| 5.3 GB/s \| ~5% \|
	3044	* \| Marvell PXA1928 (A53) \| In-order dual-issue \| 1.9 GB/s \| 1.9 GB/s \| 0% \|
	3045	*
	3046	* It also seems to fix some bad codegen on GCC, making it almost as fast as clang.
	3047	*
	3048	* @see XXH3_accumulate_512_neon()
	3049	*/
	3050	# ifndef XXH3_NEON_LANES
	3051	# if (defined(__aarch64__) \|\| defined(__arm64__) \|\| defined(_M_ARM64) \|\| defined(_M_ARM64EC)) \
	3052	&& !defined(__OPTIMIZE_SIZE__)
	3053	# define XXH3_NEON_LANES 6
	3054	# else
	3055	# define XXH3_NEON_LANES XXH_ACC_NB
	3056	# endif
	3057	# endif
	3058	#endif /* XXH_VECTOR == XXH_NEON */
	3059
	3060	/*
	3061	* VSX and Z Vector helpers.
	3062	*
	3063	* This is very messy, and any pull requests to clean this up are welcome.
	3064	*
	3065	* There are a lot of problems with supporting VSX and s390x, due to
	3066	* inconsistent intrinsics, spotty coverage, and multiple endiannesses.
	3067	*/
	3068	#if XXH_VECTOR == XXH_VSX
	3069	# if defined(__s390x__)
	3070	# include <s390intrin.h>
	3071	# else
	3072	/* gcc's altivec.h can have the unwanted consequence to unconditionally
	3073	* #define bool, vector, and pixel keywords,
	3074	* with bad consequences for programs already using these keywords for other purposes.
	3075	* The paragraph defining these macros is skipped when __APPLE_ALTIVEC__ is defined.
	3076	* __APPLE_ALTIVEC__ is _generally_ defined automatically by the compiler,
	3077	* but it seems that, in some cases, it isn't.
	3078	* Force the build macro to be defined, so that keywords are not altered.
	3079	*/
	3080	# if defined(__GNUC__) && !defined(__APPLE_ALTIVEC__)
	3081	# define __APPLE_ALTIVEC__
	3082	# endif
	3083	# include <altivec.h>
	3084	# endif
	3085
	3086	typedef __vector unsigned long long xxh_u64x2;
	3087	typedef __vector unsigned char xxh_u8x16;
	3088	typedef __vector unsigned xxh_u32x4;
	3089
	3090	# ifndef XXH_VSX_BE
	3091	# if defined(__BIG_ENDIAN__) \
	3092	\|\| (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
	3093	# define XXH_VSX_BE 1
	3094	# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__
	3095	# warning "-maltivec=be is not recommended. Please use native endianness."
	3096	# define XXH_VSX_BE 1
	3097	# else
	3098	# define XXH_VSX_BE 0
	3099	# endif
	3100	# endif /* !defined(XXH_VSX_BE) */
	3101
	3102	# if XXH_VSX_BE
	3103	# if defined(__POWER9_VECTOR__) \|\| (defined(__clang__) && defined(__s390x__))
	3104	# define XXH_vec_revb vec_revb
	3105	# else
	3106	/*!
	3107	* A polyfill for POWER9's vec_revb().
	3108	*/
	3109	XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val)
	3110	{
	3111	xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
	3112	0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 };
	3113	return vec_perm(val, val, vByteSwap);
	3114	}
	3115	# endif
	3116	# endif /* XXH_VSX_BE */
	3117
	3118	/*!
	3119	* Performs an unaligned vector load and byte swaps it on big endian.
	3120	*/
	3121	XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr)
	3122	{
	3123	xxh_u64x2 ret;
	3124	XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2));
	3125	# if XXH_VSX_BE
	3126	ret = XXH_vec_revb(ret);
	3127	# endif
	3128	return ret;
	3129	}
	3130
	3131	/*
	3132	* vec_mulo and vec_mule are very problematic intrinsics on PowerPC
	3133	*
	3134	* These intrinsics weren't added until GCC 8, despite existing for a while,
	3135	* and they are endian dependent. Also, their meaning swap depending on version.
	3136	* */
	3137	# if defined(__s390x__)
	3138	/* s390x is always big endian, no issue on this platform */
	3139	# define XXH_vec_mulo vec_mulo
	3140	# define XXH_vec_mule vec_mule
	3141	# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw)
	3142	/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */
	3143	# define XXH_vec_mulo __builtin_altivec_vmulouw
	3144	# define XXH_vec_mule __builtin_altivec_vmuleuw
	3145	# else
	3146	/* gcc needs inline assembly */
	3147	/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */
	3148	XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b)
	3149	{
	3150	xxh_u64x2 result;
	3151	__asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
	3152	return result;
	3153	}
	3154	XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b)
	3155	{
	3156	xxh_u64x2 result;
	3157	__asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
	3158	return result;
	3159	}
	3160	# endif /* XXH_vec_mulo, XXH_vec_mule */
	3161	#endif /* XXH_VECTOR == XXH_VSX */
	3162
	3163
	3164	/* prefetch
	3165	* can be disabled, by declaring XXH_NO_PREFETCH build macro */
	3166	#if defined(XXH_NO_PREFETCH)
	3167	# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */
	3168	#else
	3169	# if defined(_MSC_VER) && (defined(_M_X64) \|\| defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */
	3170	# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
	3171	# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
	3172	# elif defined(__GNUC__) && ( (__GNUC__ >= 4) \|\| ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
	3173	# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read /, 3 / locality */)
	3174	# else
	3175	# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */
	3176	# endif
	3177	#endif /* XXH_NO_PREFETCH */
	3178
	3179
	3180	/* ==========================================
	3181	* XXH3 default settings
	3182	* ========================================== */
	3183
	3184	#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */
	3185
	3186	#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN)
	3187	# error "default keyset is not large enough"
	3188	#endif
	3189
	3190	/! Pseudorandom secret taken directly from FARSH. /
	3191	XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = {
	3192	0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
	3193	0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
	3194	0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21,
	3195	0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c,
	3196	0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3,
	3197	0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8,
	3198	0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d,
	3199	0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64,
	3200	0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb,
	3201	0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e,
	3202	0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce,
	3203	0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e,
	3204	};
	3205
	3206
	3207	#ifdef XXH_OLD_NAMES
	3208	# define kSecret XXH3_kSecret
	3209	#endif
	3210
	3211	#ifdef XXH_DOXYGEN
	3212	/*!
	3213	* @brief Calculates a 32-bit to 64-bit long multiply.
	3214	*
	3215	* Implemented as a macro.
	3216	*
	3217	* Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't
	3218	* need to (but it shouldn't need to anyways, it is about 7 instructions to do
	3219	* a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we
	3220	* use that instead of the normal method.
	3221	*
	3222	* If you are compiling for platforms like Thumb-1 and don't have a better option,
	3223	* you may also want to write your own long multiply routine here.
	3224	*
	3225	* @param x, y Numbers to be multiplied
	3226	* @return 64-bit product of the low 32 bits of @p x and @p y.
	3227	*/
	3228	XXH_FORCE_INLINE xxh_u64
	3229	XXH_mult32to64(xxh_u64 x, xxh_u64 y)
	3230	{
	3231	return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF);
	3232	}
	3233	#elif defined(_MSC_VER) && defined(_M_IX86)
	3234	# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y))
	3235	#else
	3236	/*
	3237	* Downcast + upcast is usually better than masking on older compilers like
	3238	* GCC 4.2 (especially 32-bit ones), all without affecting newer compilers.
	3239	*
	3240	* The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands
	3241	* and perform a full 64x64 multiply -- entirely redundant on 32-bit.
	3242	*/
	3243	# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y))
	3244	#endif
	3245
	3246	/*!
	3247	* @brief Calculates a 64->128-bit long multiply.
	3248	*
	3249	* Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar
	3250	* version.
	3251	*
	3252	* @param lhs , rhs The 64-bit integers to be multiplied
	3253	* @return The 128-bit result represented in an @ref XXH128_hash_t.
	3254	*/
	3255	static XXH128_hash_t
	3256	XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs)
	3257	{
	3258	/*
	3259	* GCC/Clang __uint128_t method.
	3260	*
	3261	* On most 64-bit targets, GCC and Clang define a __uint128_t type.
	3262	* This is usually the best way as it usually uses a native long 64-bit
	3263	* multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64.
	3264	*
	3265	* Usually.
	3266	*
	3267	* Despite being a 32-bit platform, Clang (and emscripten) define this type
	3268	* despite not having the arithmetic for it. This results in a laggy
	3269	* compiler builtin call which calculates a full 128-bit multiply.
	3270	* In that case it is best to use the portable one.
	3271	* https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677
	3272	*/
	3273	#if (defined(__GNUC__) \|\| defined(__clang__)) && !defined(__wasm__) \
	3274	&& defined(__SIZEOF_INT128__) \
	3275	\|\| (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
	3276
	3277	__uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs;
	3278	XXH128_hash_t r128;
	3279	r128.low64 = (xxh_u64)(product);
	3280	r128.high64 = (xxh_u64)(product >> 64);
	3281	return r128;
	3282
	3283	/*
	3284	* MSVC for x64's _umul128 method.
	3285	*
	3286	* xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct);
	3287	*
	3288	* This compiles to single operand MUL on x64.
	3289	*/
	3290	#elif (defined(_M_X64) \|\| defined(_M_IA64)) && !defined(_M_ARM64EC)
	3291
	3292	#ifndef _MSC_VER
	3293	# pragma intrinsic(_umul128)
	3294	#endif
	3295	xxh_u64 product_high;
	3296	xxh_u64 const product_low = _umul128(lhs, rhs, &product_high);
	3297	XXH128_hash_t r128;
	3298	r128.low64 = product_low;
	3299	r128.high64 = product_high;
	3300	return r128;
	3301
	3302	/*
	3303	* MSVC for ARM64's __umulh method.
	3304	*
	3305	* This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method.
	3306	*/
	3307	#elif defined(_M_ARM64) \|\| defined(_M_ARM64EC)
	3308
	3309	#ifndef _MSC_VER
	3310	# pragma intrinsic(__umulh)
	3311	#endif
	3312	XXH128_hash_t r128;
	3313	r128.low64 = lhs * rhs;
	3314	r128.high64 = __umulh(lhs, rhs);
	3315	return r128;
	3316
	3317	#else
	3318	/*
	3319	* Portable scalar method. Optimized for 32-bit and 64-bit ALUs.
	3320	*
	3321	* This is a fast and simple grade school multiply, which is shown below
	3322	* with base 10 arithmetic instead of base 0x100000000.
	3323	*
	3324	* 9 3 // D2 lhs = 93
	3325	* x 7 5 // D2 rhs = 75
	3326	* ----------
	3327	* 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15
	3328	* 4 5 \| // D2 hi_lo = (93 / 10) * (75 % 10) = 45
	3329	* 2 1 \| // D2 lo_hi = (93 % 10) * (75 / 10) = 21
	3330	* + 6 3 \| \| // D2 hi_hi = (93 / 10) * (75 / 10) = 63
	3331	* ---------
	3332	* 2 7 \| // D2 cross = (15 / 10) + (45 % 10) + 21 = 27
	3333	* + 6 7 \| \| // D2 upper = (27 / 10) + (45 / 10) + 63 = 67
	3334	* ---------
	3335	* 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975
	3336	*
	3337	* The reasons for adding the products like this are:
	3338	* 1. It avoids manual carry tracking. Just like how
	3339	* (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX.
	3340	* This avoids a lot of complexity.
	3341	*
	3342	* 2. It hints for, and on Clang, compiles to, the powerful UMAAL
	3343	* instruction available in ARM's Digital Signal Processing extension
	3344	* in 32-bit ARMv6 and later, which is shown below:
	3345	*
	3346	* void UMAAL(xxh_u32 RdLo, xxh_u32 RdHi, xxh_u32 Rn, xxh_u32 Rm)
	3347	* {
	3348	* xxh_u64 product = (xxh_u64)RdLo (xxh_u64)*RdHi + Rn + Rm;
	3349	* *RdLo = (xxh_u32)(product & 0xFFFFFFFF);
	3350	* *RdHi = (xxh_u32)(product >> 32);
	3351	* }
	3352	*
	3353	* This instruction was designed for efficient long multiplication, and
	3354	* allows this to be calculated in only 4 instructions at speeds
	3355	* comparable to some 64-bit ALUs.
	3356	*
	3357	* 3. It isn't terrible on other platforms. Usually this will be a couple
	3358	* of 32-bit ADD/ADCs.
	3359	*/
	3360
	3361	/* First calculate all of the cross products. */
	3362	xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF);
	3363	xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF);
	3364	xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32);
	3365	xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32);
	3366
	3367	/* Now add the products together. These will never overflow. */
	3368	xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi;
	3369	xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi;
	3370	xxh_u64 const lower = (cross << 32) \| (lo_lo & 0xFFFFFFFF);
	3371
	3372	XXH128_hash_t r128;
	3373	r128.low64 = lower;
	3374	r128.high64 = upper;
	3375	return r128;
	3376	#endif
	3377	}
	3378
	3379	/*!
	3380	* @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it.
	3381	*
	3382	* The reason for the separate function is to prevent passing too many structs
	3383	* around by value. This will hopefully inline the multiply, but we don't force it.
	3384	*
	3385	* @param lhs , rhs The 64-bit integers to multiply
	3386	* @return The low 64 bits of the product XOR'd by the high 64 bits.
	3387	* @see XXH_mult64to128()
	3388	*/
	3389	static xxh_u64
	3390	XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs)
	3391	{
	3392	XXH128_hash_t product = XXH_mult64to128(lhs, rhs);
	3393	return product.low64 ^ product.high64;
	3394	}
	3395
	3396	/! Seems to produce slightly better code on GCC for some reason. /
	3397	XXH_FORCE_INLINE xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift)
	3398	{
	3399	XXH_ASSERT(0 <= shift && shift < 64);
	3400	return v64 ^ (v64 >> shift);
	3401	}
	3402
	3403	/*
	3404	* This is a fast avalanche stage,
	3405	* suitable when input bits are already partially mixed
	3406	*/
	3407	static XXH64_hash_t XXH3_avalanche(xxh_u64 h64)
	3408	{
	3409	h64 = XXH_xorshift64(h64, 37);
	3410	h64 *= 0x165667919E3779F9ULL;
	3411	h64 = XXH_xorshift64(h64, 32);
	3412	return h64;
	3413	}
	3414
	3415	/*
	3416	* This is a stronger avalanche,
	3417	* inspired by Pelle Evensen's rrmxmx
	3418	* preferable when input has not been previously mixed
	3419	*/
	3420	static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len)
	3421	{
	3422	/* this mix is inspired by Pelle Evensen's rrmxmx */
	3423	h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24);
	3424	h64 *= 0x9FB21C651E98DF25ULL;
	3425	h64 ^= (h64 >> 35) + len ;
	3426	h64 *= 0x9FB21C651E98DF25ULL;
	3427	return XXH_xorshift64(h64, 28);
	3428	}
	3429
	3430
	3431	/* ==========================================
	3432	* Short keys
	3433	* ==========================================
	3434	* One of the shortcomings of XXH32 and XXH64 was that their performance was
	3435	* sub-optimal on short lengths. It used an iterative algorithm which strongly
	3436	* favored lengths that were a multiple of 4 or 8.
	3437	*
	3438	* Instead of iterating over individual inputs, we use a set of single shot
	3439	* functions which piece together a range of lengths and operate in constant time.
	3440	*
	3441	* Additionally, the number of multiplies has been significantly reduced. This
	3442	* reduces latency, especially when emulating 64-bit multiplies on 32-bit.
	3443	*
	3444	* Depending on the platform, this may or may not be faster than XXH32, but it
	3445	* is almost guaranteed to be faster than XXH64.
	3446	*/
	3447
	3448	/*
	3449	* At very short lengths, there isn't enough input to fully hide secrets, or use
	3450	* the entire secret.
	3451	*
	3452	* There is also only a limited amount of mixing we can do before significantly
	3453	* impacting performance.
	3454	*
	3455	* Therefore, we use different sections of the secret and always mix two secret
	3456	* samples with an XOR. This should have no effect on performance on the
	3457	* seedless or withSeed variants because everything _should_ be constant folded
	3458	* by modern compilers.
	3459	*
	3460	* The XOR mixing hides individual parts of the secret and increases entropy.
	3461	*
	3462	* This adds an extra layer of strength for custom secrets.
	3463	*/
	3464	XXH_FORCE_INLINE XXH64_hash_t
	3465	XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
	3466	{
	3467	XXH_ASSERT(input != NULL);
	3468	XXH_ASSERT(1 <= len && len <= 3);
	3469	XXH_ASSERT(secret != NULL);
	3470	/*
	3471	* len = 1: combined = { input[0], 0x01, input[0], input[0] }
	3472	* len = 2: combined = { input[1], 0x02, input[0], input[1] }
	3473	* len = 3: combined = { input[2], 0x03, input[0], input[1] }
	3474	*/
	3475	{ xxh_u8 const c1 = input[0];
	3476	xxh_u8 const c2 = input[len >> 1];
	3477	xxh_u8 const c3 = input[len - 1];
	3478	xxh_u32 const combined = ((xxh_u32)c1 << 16) \| ((xxh_u32)c2 << 24)
	3479	\| ((xxh_u32)c3 << 0) \| ((xxh_u32)len << 8);
	3480	xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed;
	3481	xxh_u64 const keyed = (xxh_u64)combined ^ bitflip;
	3482	return XXH64_avalanche(keyed);
	3483	}
	3484	}
	3485
	3486	XXH_FORCE_INLINE XXH64_hash_t
	3487	XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
	3488	{
	3489	XXH_ASSERT(input != NULL);
	3490	XXH_ASSERT(secret != NULL);
	3491	XXH_ASSERT(4 <= len && len <= 8);
	3492	seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32;
	3493	{ xxh_u32 const input1 = XXH_readLE32(input);
	3494	xxh_u32 const input2 = XXH_readLE32(input + len - 4);
	3495	xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed;
	3496	xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32);
	3497	xxh_u64 const keyed = input64 ^ bitflip;
	3498	return XXH3_rrmxmx(keyed, len);
	3499	}
	3500	}
	3501
	3502	XXH_FORCE_INLINE XXH64_hash_t
	3503	XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
	3504	{
	3505	XXH_ASSERT(input != NULL);
	3506	XXH_ASSERT(secret != NULL);
	3507	XXH_ASSERT(9 <= len && len <= 16);
	3508	{ xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed;
	3509	xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed;
	3510	xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1;
	3511	xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2;
	3512	xxh_u64 const acc = len
	3513	+ XXH_swap64(input_lo) + input_hi
	3514	+ XXH3_mul128_fold64(input_lo, input_hi);
	3515	return XXH3_avalanche(acc);
	3516	}
	3517	}
	3518
	3519	XXH_FORCE_INLINE XXH64_hash_t
	3520	XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
	3521	{
	3522	XXH_ASSERT(len <= 16);
	3523	{ if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed);
	3524	if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed);
	3525	if (len) return XXH3_len_1to3_64b(input, len, secret, seed);
	3526	return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64)));
	3527	}
	3528	}
	3529
	3530	/*
	3531	* DISCLAIMER: There are known seed-dependent multicollisions here due to
	3532	* multiplication by zero, affecting hashes of lengths 17 to 240.
	3533	*
	3534	* However, they are very unlikely.
	3535	*
	3536	* Keep this in mind when using the unseeded XXH3_64bits() variant: As with all
	3537	* unseeded non-cryptographic hashes, it does not attempt to defend itself
	3538	* against specially crafted inputs, only random inputs.
	3539	*
	3540	* Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes
	3541	* cancelling out the secret is taken an arbitrary number of times (addressed
	3542	* in XXH3_accumulate_512), this collision is very unlikely with random inputs
	3543	* and/or proper seeding:
	3544	*
	3545	* This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a
	3546	* function that is only called up to 16 times per hash with up to 240 bytes of
	3547	* input.
	3548	*
	3549	* This is not too bad for a non-cryptographic hash function, especially with
	3550	* only 64 bit outputs.
	3551	*
	3552	* The 128-bit variant (which trades some speed for strength) is NOT affected
	3553	* by this, although it is always a good idea to use a proper seed if you care
	3554	* about strength.
	3555	*/
	3556	XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input,
	3557	const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64)
	3558	{
	3559	#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
	3560	&& defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \
	3561	&& !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */
	3562	/*
	3563	* UGLY HACK:
	3564	* GCC for x86 tends to autovectorize the 128-bit multiply, resulting in
	3565	* slower code.
	3566	*
	3567	* By forcing seed64 into a register, we disrupt the cost model and
	3568	* cause it to scalarize. See `XXH32_round()`
	3569	*
	3570	* FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600,
	3571	* XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on
	3572	* GCC 9.2, despite both emitting scalar code.
	3573	*
	3574	* GCC generates much better scalar code than Clang for the rest of XXH3,
	3575	* which is why finding a more optimal codepath is an interest.
	3576	*/
	3577	XXH_COMPILER_GUARD(seed64);
	3578	#endif
	3579	{ xxh_u64 const input_lo = XXH_readLE64(input);
	3580	xxh_u64 const input_hi = XXH_readLE64(input+8);
	3581	return XXH3_mul128_fold64(
	3582	input_lo ^ (XXH_readLE64(secret) + seed64),
	3583	input_hi ^ (XXH_readLE64(secret+8) - seed64)
	3584	);
	3585	}
	3586	}
	3587
	3588	/* For mid range keys, XXH3 uses a Mum-hash variant. */
	3589	XXH_FORCE_INLINE XXH64_hash_t
	3590	XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
	3591	const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
	3592	XXH64_hash_t seed)
	3593	{
	3594	XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
	3595	XXH_ASSERT(16 < len && len <= 128);
	3596
	3597	{ xxh_u64 acc = len * XXH_PRIME64_1;
	3598	if (len > 32) {
	3599	if (len > 64) {
	3600	if (len > 96) {
	3601	acc += XXH3_mix16B(input+48, secret+96, seed);
	3602	acc += XXH3_mix16B(input+len-64, secret+112, seed);
	3603	}
	3604	acc += XXH3_mix16B(input+32, secret+64, seed);
	3605	acc += XXH3_mix16B(input+len-48, secret+80, seed);
	3606	}
	3607	acc += XXH3_mix16B(input+16, secret+32, seed);
	3608	acc += XXH3_mix16B(input+len-32, secret+48, seed);
	3609	}
	3610	acc += XXH3_mix16B(input+0, secret+0, seed);
	3611	acc += XXH3_mix16B(input+len-16, secret+16, seed);
	3612
	3613	return XXH3_avalanche(acc);
	3614	}
	3615	}
	3616
	3617	#define XXH3_MIDSIZE_MAX 240
	3618
	3619	XXH_NO_INLINE XXH64_hash_t
	3620	XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
	3621	const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
	3622	XXH64_hash_t seed)
	3623	{
	3624	XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
	3625	XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
	3626
	3627	#define XXH3_MIDSIZE_STARTOFFSET 3
	3628	#define XXH3_MIDSIZE_LASTOFFSET 17
	3629
	3630	{ xxh_u64 acc = len * XXH_PRIME64_1;
	3631	int const nbRounds = (int)len / 16;
	3632	int i;
	3633	for (i=0; i<8; i++) {
	3634	acc += XXH3_mix16B(input+(16i), secret+(16i), seed);
	3635	}
	3636	acc = XXH3_avalanche(acc);
	3637	XXH_ASSERT(nbRounds >= 8);
	3638	#if defined(__clang__) /* Clang */ \
	3639	&& (defined(__ARM_NEON) \|\| defined(__ARM_NEON__)) /* NEON */ \
	3640	&& !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */
	3641	/*
	3642	* UGLY HACK:
	3643	* Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86.
	3644	* In everywhere else, it uses scalar code.
	3645	*
	3646	* For 64->128-bit multiplies, even if the NEON was 100% optimal, it
	3647	* would still be slower than UMAAL (see XXH_mult64to128).
	3648	*
	3649	* Unfortunately, Clang doesn't handle the long multiplies properly and
	3650	* converts them to the nonexistent "vmulq_u64" intrinsic, which is then
	3651	* scalarized into an ugly mess of VMOV.32 instructions.
	3652	*
	3653	* This mess is difficult to avoid without turning autovectorization
	3654	* off completely, but they are usually relatively minor and/or not
	3655	* worth it to fix.
	3656	*
	3657	* This loop is the easiest to fix, as unlike XXH32, this pragma
	3658	* _actually works_ because it is a loop vectorization instead of an
	3659	* SLP vectorization.
	3660	*/
	3661	#pragma clang loop vectorize(disable)
	3662	#endif
	3663	for (i=8 ; i < nbRounds; i++) {
	3664	acc += XXH3_mix16B(input+(16i), secret+(16(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed);
	3665	}
	3666	/* last bytes */
	3667	acc += XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed);
	3668	return XXH3_avalanche(acc);
	3669	}
	3670	}
	3671
	3672
	3673	/* ======= Long Keys ======= */
	3674
	3675	#define XXH_STRIPE_LEN 64
	3676	#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */
	3677	#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64))
	3678
	3679	#ifdef XXH_OLD_NAMES
	3680	# define STRIPE_LEN XXH_STRIPE_LEN
	3681	# define ACC_NB XXH_ACC_NB
	3682	#endif
	3683
	3684	XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64)
	3685	{
	3686	if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64);
	3687	XXH_memcpy(dst, &v64, sizeof(v64));
	3688	}
	3689
	3690	/* Several intrinsic functions below are supposed to accept __int64 as argument,
	3691	* as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ .
	3692	* However, several environments do not define __int64 type,
	3693	* requiring a workaround.
	3694	*/
	3695	#if !defined (__VMS) \
	3696	&& (defined (__cplusplus) \
	3697	\|\| (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
	3698	typedef int64_t xxh_i64;
	3699	#else
	3700	/* the following type must have a width of 64-bit */
	3701	typedef long long xxh_i64;
	3702	#endif
	3703
	3704
	3705	/*
	3706	* XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized.
	3707	*
	3708	* It is a hardened version of UMAC, based off of FARSH's implementation.
	3709	*
	3710	* This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD
	3711	* implementations, and it is ridiculously fast.
	3712	*
	3713	* We harden it by mixing the original input to the accumulators as well as the product.
	3714	*
	3715	* This means that in the (relatively likely) case of a multiply by zero, the
	3716	* original input is preserved.
	3717	*
	3718	* On 128-bit inputs, we swap 64-bit pairs when we add the input to improve
	3719	* cross-pollination, as otherwise the upper and lower halves would be
	3720	* essentially independent.
	3721	*
	3722	* This doesn't matter on 64-bit hashes since they all get merged together in
	3723	* the end, so we skip the extra step.
	3724	*
	3725	* Both XXH3_64bits and XXH3_128bits use this subroutine.
	3726	*/
	3727
	3728	#if (XXH_VECTOR == XXH_AVX512) \
	3729	\|\| (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0)
	3730
	3731	#ifndef XXH_TARGET_AVX512
	3732	# define XXH_TARGET_AVX512 /* disable attribute target */
	3733	#endif
	3734
	3735	XXH_FORCE_INLINE XXH_TARGET_AVX512 void
	3736	XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc,
	3737	const void* XXH_RESTRICT input,
	3738	const void* XXH_RESTRICT secret)
	3739	{
	3740	__m512i* const xacc = (__m512i *) acc;
	3741	XXH_ASSERT((((size_t)acc) & 63) == 0);
	3742	XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i));
	3743
	3744	{
	3745	/* data_vec = input[0]; */
	3746	__m512i const data_vec = _mm512_loadu_si512 (input);
	3747	/* key_vec = secret[0]; */
	3748	__m512i const key_vec = _mm512_loadu_si512 (secret);
	3749	/* data_key = data_vec ^ key_vec; */
	3750	__m512i const data_key = _mm512_xor_si512 (data_vec, key_vec);
	3751	/* data_key_lo = data_key >> 32; */
	3752	__m512i const data_key_lo = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1));
	3753	/* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
	3754	__m512i const product = _mm512_mul_epu32 (data_key, data_key_lo);
	3755	/* xacc[0] += swap(data_vec); */
	3756	__m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2));
	3757	__m512i const sum = _mm512_add_epi64(*xacc, data_swap);
	3758	/* xacc[0] += product; */
	3759	*xacc = _mm512_add_epi64(product, sum);
	3760	}
	3761	}
	3762
	3763	/*
	3764	* XXH3_scrambleAcc: Scrambles the accumulators to improve mixing.
	3765	*
	3766	* Multiplication isn't perfect, as explained by Google in HighwayHash:
	3767	*
	3768	* // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to
	3769	* // varying degrees. In descending order of goodness, bytes
	3770	* // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32.
	3771	* // As expected, the upper and lower bytes are much worse.
	3772	*
	3773	* Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291
	3774	*
	3775	* Since our algorithm uses a pseudorandom secret to add some variance into the
	3776	* mix, we don't need to (or want to) mix as often or as much as HighwayHash does.
	3777	*
	3778	* This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid
	3779	* extraction.
	3780	*
	3781	* Both XXH3_64bits and XXH3_128bits use this subroutine.
	3782	*/
	3783
	3784	XXH_FORCE_INLINE XXH_TARGET_AVX512 void
	3785	XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
	3786	{
	3787	XXH_ASSERT((((size_t)acc) & 63) == 0);
	3788	XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i));
	3789	{ __m512i* const xacc = (__m512i*) acc;
	3790	const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1);
	3791
	3792	/* xacc[0] ^= (xacc[0] >> 47) */
	3793	__m512i const acc_vec = *xacc;
	3794	__m512i const shifted = _mm512_srli_epi64 (acc_vec, 47);
	3795	__m512i const data_vec = _mm512_xor_si512 (acc_vec, shifted);
	3796	/* xacc[0] ^= secret; */
	3797	__m512i const key_vec = _mm512_loadu_si512 (secret);
	3798	__m512i const data_key = _mm512_xor_si512 (data_vec, key_vec);
	3799
	3800	/* xacc[0] = XXH_PRIME32_1; /
	3801	__m512i const data_key_hi = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1));
	3802	__m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32);
	3803	__m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32);
	3804	*xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32));
	3805	}
	3806	}
	3807
	3808	XXH_FORCE_INLINE XXH_TARGET_AVX512 void
	3809	XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
	3810	{
	3811	XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0);
	3812	XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64);
	3813	XXH_ASSERT(((size_t)customSecret & 63) == 0);
	3814	(void)(&XXH_writeLE64);
	3815	{ int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i);
	3816	__m512i const seed = _mm512_mask_set1_epi64(_mm512_set1_epi64((xxh_i64)seed64), 0xAA, (xxh_i64)(0U - seed64));
	3817
	3818	const __m512i* const src = (const __m512i) ((const void) XXH3_kSecret);
	3819	__m512i* const dest = ( __m512i*) customSecret;
	3820	int i;
	3821	XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */
	3822	XXH_ASSERT(((size_t)dest & 63) == 0);
	3823	for (i=0; i < nbRounds; ++i) {
	3824	/* GCC has a bug, _mm512_stream_load_si512 accepts 'void', not 'void const',
	3825	* this will warn "discards 'const' qualifier". */
	3826	union {
	3827	const __m512i* cp;
	3828	void* p;
	3829	} remote_const_void;
	3830	remote_const_void.cp = src + i;
	3831	dest[i] = _mm512_add_epi64(_mm512_stream_load_si512(remote_const_void.p), seed);
	3832	} }
	3833	}
	3834
	3835	#endif
	3836
	3837	#if (XXH_VECTOR == XXH_AVX2) \
	3838	\|\| (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0)
	3839
	3840	#ifndef XXH_TARGET_AVX2
	3841	# define XXH_TARGET_AVX2 /* disable attribute target */
	3842	#endif
	3843
	3844	XXH_FORCE_INLINE XXH_TARGET_AVX2 void
	3845	XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc,
	3846	const void* XXH_RESTRICT input,
	3847	const void* XXH_RESTRICT secret)
	3848	{
	3849	XXH_ASSERT((((size_t)acc) & 31) == 0);
	3850	{ __m256i* const xacc = (__m256i *) acc;
	3851	/* Unaligned. This is mainly for pointer arithmetic, and because
	3852	* _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
	3853	const __m256i* const xinput = (const __m256i *) input;
	3854	/* Unaligned. This is mainly for pointer arithmetic, and because
	3855	* _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
	3856	const __m256i* const xsecret = (const __m256i *) secret;
	3857
	3858	size_t i;
	3859	for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) {
	3860	/* data_vec = xinput[i]; */
	3861	__m256i const data_vec = _mm256_loadu_si256 (xinput+i);
	3862	/* key_vec = xsecret[i]; */
	3863	__m256i const key_vec = _mm256_loadu_si256 (xsecret+i);
	3864	/* data_key = data_vec ^ key_vec; */
	3865	__m256i const data_key = _mm256_xor_si256 (data_vec, key_vec);
	3866	/* data_key_lo = data_key >> 32; */
	3867	__m256i const data_key_lo = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
	3868	/* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
	3869	__m256i const product = _mm256_mul_epu32 (data_key, data_key_lo);
	3870	/* xacc[i] += swap(data_vec); */
	3871	__m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2));
	3872	__m256i const sum = _mm256_add_epi64(xacc[i], data_swap);
	3873	/* xacc[i] += product; */
	3874	xacc[i] = _mm256_add_epi64(product, sum);
	3875	} }
	3876	}
	3877
	3878	XXH_FORCE_INLINE XXH_TARGET_AVX2 void
	3879	XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
	3880	{
	3881	XXH_ASSERT((((size_t)acc) & 31) == 0);
	3882	{ __m256i* const xacc = (__m256i*) acc;
	3883	/* Unaligned. This is mainly for pointer arithmetic, and because
	3884	* _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
	3885	const __m256i* const xsecret = (const __m256i *) secret;
	3886	const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1);
	3887
	3888	size_t i;
	3889	for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) {
	3890	/* xacc[i] ^= (xacc[i] >> 47) */
	3891	__m256i const acc_vec = xacc[i];
	3892	__m256i const shifted = _mm256_srli_epi64 (acc_vec, 47);
	3893	__m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted);
	3894	/* xacc[i] ^= xsecret; */
	3895	__m256i const key_vec = _mm256_loadu_si256 (xsecret+i);
	3896	__m256i const data_key = _mm256_xor_si256 (data_vec, key_vec);
	3897
	3898	/* xacc[i] = XXH_PRIME32_1; /
	3899	__m256i const data_key_hi = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
	3900	__m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32);
	3901	__m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32);
	3902	xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32));
	3903	}
	3904	}
	3905	}
	3906
	3907	XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
	3908	{
	3909	XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0);
	3910	XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6);
	3911	XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64);
	3912	(void)(&XXH_writeLE64);
	3913	XXH_PREFETCH(customSecret);
	3914	{ __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64);
	3915
	3916	const __m256i* const src = (const __m256i) ((const void) XXH3_kSecret);
	3917	__m256i* dest = ( __m256i*) customSecret;
	3918
	3919	# if defined(__GNUC__) \|\| defined(__clang__)
	3920	/*
	3921	* On GCC & Clang, marking 'dest' as modified will cause the compiler:
	3922	* - do not extract the secret from sse registers in the internal loop
	3923	* - use less common registers, and avoid pushing these reg into stack
	3924	*/
	3925	XXH_COMPILER_GUARD(dest);
	3926	# endif
	3927	XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */
	3928	XXH_ASSERT(((size_t)dest & 31) == 0);
	3929
	3930	/* GCC -O2 need unroll loop manually */
	3931	dest[0] = _mm256_add_epi64(_mm256_stream_load_si256(src+0), seed);
	3932	dest[1] = _mm256_add_epi64(_mm256_stream_load_si256(src+1), seed);
	3933	dest[2] = _mm256_add_epi64(_mm256_stream_load_si256(src+2), seed);
	3934	dest[3] = _mm256_add_epi64(_mm256_stream_load_si256(src+3), seed);
	3935	dest[4] = _mm256_add_epi64(_mm256_stream_load_si256(src+4), seed);
	3936	dest[5] = _mm256_add_epi64(_mm256_stream_load_si256(src+5), seed);
	3937	}
	3938	}
	3939
	3940	#endif
	3941
	3942	/* x86dispatch always generates SSE2 */
	3943	#if (XXH_VECTOR == XXH_SSE2) \|\| defined(XXH_X86DISPATCH)
	3944
	3945	#ifndef XXH_TARGET_SSE2
	3946	# define XXH_TARGET_SSE2 /* disable attribute target */
	3947	#endif
	3948
	3949	XXH_FORCE_INLINE XXH_TARGET_SSE2 void
	3950	XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc,
	3951	const void* XXH_RESTRICT input,
	3952	const void* XXH_RESTRICT secret)
	3953	{
	3954	/* SSE2 is just a half-scale version of the AVX2 version. */
	3955	XXH_ASSERT((((size_t)acc) & 15) == 0);
	3956	{ __m128i* const xacc = (__m128i *) acc;
	3957	/* Unaligned. This is mainly for pointer arithmetic, and because
	3958	* _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
	3959	const __m128i* const xinput = (const __m128i *) input;
	3960	/* Unaligned. This is mainly for pointer arithmetic, and because
	3961	* _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
	3962	const __m128i* const xsecret = (const __m128i *) secret;
	3963
	3964	size_t i;
	3965	for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) {
	3966	/* data_vec = xinput[i]; */
	3967	__m128i const data_vec = _mm_loadu_si128 (xinput+i);
	3968	/* key_vec = xsecret[i]; */
	3969	__m128i const key_vec = _mm_loadu_si128 (xsecret+i);
	3970	/* data_key = data_vec ^ key_vec; */
	3971	__m128i const data_key = _mm_xor_si128 (data_vec, key_vec);
	3972	/* data_key_lo = data_key >> 32; */
	3973	__m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
	3974	/* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
	3975	__m128i const product = _mm_mul_epu32 (data_key, data_key_lo);
	3976	/* xacc[i] += swap(data_vec); */
	3977	__m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2));
	3978	__m128i const sum = _mm_add_epi64(xacc[i], data_swap);
	3979	/* xacc[i] += product; */
	3980	xacc[i] = _mm_add_epi64(product, sum);
	3981	} }
	3982	}
	3983
	3984	XXH_FORCE_INLINE XXH_TARGET_SSE2 void
	3985	XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
	3986	{
	3987	XXH_ASSERT((((size_t)acc) & 15) == 0);
	3988	{ __m128i* const xacc = (__m128i*) acc;
	3989	/* Unaligned. This is mainly for pointer arithmetic, and because
	3990	* _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
	3991	const __m128i* const xsecret = (const __m128i *) secret;
	3992	const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1);
	3993
	3994	size_t i;
	3995	for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) {
	3996	/* xacc[i] ^= (xacc[i] >> 47) */
	3997	__m128i const acc_vec = xacc[i];
	3998	__m128i const shifted = _mm_srli_epi64 (acc_vec, 47);
	3999	__m128i const data_vec = _mm_xor_si128 (acc_vec, shifted);
	4000	/* xacc[i] ^= xsecret[i]; */
	4001	__m128i const key_vec = _mm_loadu_si128 (xsecret+i);
	4002	__m128i const data_key = _mm_xor_si128 (data_vec, key_vec);
	4003
	4004	/* xacc[i] = XXH_PRIME32_1; /
	4005	__m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
	4006	__m128i const prod_lo = _mm_mul_epu32 (data_key, prime32);
	4007	__m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32);
	4008	xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32));
	4009	}
	4010	}
	4011	}
	4012
	4013	XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
	4014	{
	4015	XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);
	4016	(void)(&XXH_writeLE64);
	4017	{ int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i);
	4018
	4019	# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900
	4020	/* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */
	4021	XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) };
	4022	__m128i const seed = _mm_load_si128((__m128i const*)seed64x2);
	4023	# else
	4024	__m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64);
	4025	# endif
	4026	int i;
	4027
	4028	const void* const src16 = XXH3_kSecret;
	4029	__m128i* dst16 = (__m128i*) customSecret;
	4030	# if defined(__GNUC__) \|\| defined(__clang__)
	4031	/*
	4032	* On GCC & Clang, marking 'dest' as modified will cause the compiler:
	4033	* - do not extract the secret from sse registers in the internal loop
	4034	* - use less common registers, and avoid pushing these reg into stack
	4035	*/
	4036	XXH_COMPILER_GUARD(dst16);
	4037	# endif
	4038	XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */
	4039	XXH_ASSERT(((size_t)dst16 & 15) == 0);
	4040
	4041	for (i=0; i < nbRounds; ++i) {
	4042	dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed);
	4043	} }
	4044	}
	4045
	4046	#endif
	4047
	4048	#if (XXH_VECTOR == XXH_NEON)
	4049
	4050	/* forward declarations for the scalar routines */
	4051	XXH_FORCE_INLINE void
	4052	XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input,
	4053	void const* XXH_RESTRICT secret, size_t lane);
	4054
	4055	XXH_FORCE_INLINE void
	4056	XXH3_scalarScrambleRound(void* XXH_RESTRICT acc,
	4057	void const* XXH_RESTRICT secret, size_t lane);
	4058
	4059	/*!
	4060	* @internal
	4061	* @brief The bulk processing loop for NEON.
	4062	*
	4063	* The NEON code path is actually partially scalar when running on AArch64. This
	4064	* is to optimize the pipelining and can have up to 15% speedup depending on the
	4065	* CPU, and it also mitigates some GCC codegen issues.
	4066	*
	4067	* @see XXH3_NEON_LANES for configuring this and details about this optimization.
	4068	*/
	4069	XXH_FORCE_INLINE void
	4070	XXH3_accumulate_512_neon( void* XXH_RESTRICT acc,
	4071	const void* XXH_RESTRICT input,
	4072	const void* XXH_RESTRICT secret)
	4073	{
	4074	XXH_ASSERT((((size_t)acc) & 15) == 0);
	4075	XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0);
	4076	{
	4077	uint64x2_t* const xacc = (uint64x2_t *) acc;
	4078	/* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */
	4079	uint8_t const* const xinput = (const uint8_t *) input;
	4080	uint8_t const* const xsecret = (const uint8_t *) secret;
	4081
	4082	size_t i;
	4083	/* NEON for the first few lanes (these loops are normally interleaved) */
	4084	for (i=0; i < XXH3_NEON_LANES / 2; i++) {
	4085	/* data_vec = xinput[i]; */
	4086	uint8x16_t data_vec = vld1q_u8(xinput + (i * 16));
	4087	/* key_vec = xsecret[i]; */
	4088	uint8x16_t key_vec = vld1q_u8(xsecret + (i * 16));
	4089	uint64x2_t data_key;
	4090	uint32x2_t data_key_lo, data_key_hi;
	4091	/* xacc[i] += swap(data_vec); */
	4092	uint64x2_t const data64 = vreinterpretq_u64_u8(data_vec);
	4093	uint64x2_t const swapped = vextq_u64(data64, data64, 1);
	4094	xacc[i] = vaddq_u64 (xacc[i], swapped);
	4095	/* data_key = data_vec ^ key_vec; */
	4096	data_key = vreinterpretq_u64_u8(veorq_u8(data_vec, key_vec));
	4097	/* data_key_lo = (uint32x2_t) (data_key & 0xFFFFFFFF);
	4098	* data_key_hi = (uint32x2_t) (data_key >> 32);
	4099	* data_key = UNDEFINED; */
	4100	XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi);
	4101	/* xacc[i] += (uint64x2_t) data_key_lo * (uint64x2_t) data_key_hi; */
	4102	xacc[i] = vmlal_u32 (xacc[i], data_key_lo, data_key_hi);
	4103
	4104	}
	4105	/* Scalar for the remainder. This may be a zero iteration loop. */
	4106	for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) {
	4107	XXH3_scalarRound(acc, input, secret, i);
	4108	}
	4109	}
	4110	}
	4111
	4112	XXH_FORCE_INLINE void
	4113	XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
	4114	{
	4115	XXH_ASSERT((((size_t)acc) & 15) == 0);
	4116
	4117	{ uint64x2_t* xacc = (uint64x2_t*) acc;
	4118	uint8_t const* xsecret = (uint8_t const*) secret;
	4119	uint32x2_t prime = vdup_n_u32 (XXH_PRIME32_1);
	4120
	4121	size_t i;
	4122	/* NEON for the first few lanes (these loops are normally interleaved) */
	4123	for (i=0; i < XXH3_NEON_LANES / 2; i++) {
	4124	/* xacc[i] ^= (xacc[i] >> 47); */
	4125	uint64x2_t acc_vec = xacc[i];
	4126	uint64x2_t shifted = vshrq_n_u64 (acc_vec, 47);
	4127	uint64x2_t data_vec = veorq_u64 (acc_vec, shifted);
	4128
	4129	/* xacc[i] ^= xsecret[i]; */
	4130	uint8x16_t key_vec = vld1q_u8 (xsecret + (i * 16));
	4131	uint64x2_t data_key = veorq_u64 (data_vec, vreinterpretq_u64_u8(key_vec));
	4132
	4133	/* xacc[i] = XXH_PRIME32_1 /
	4134	uint32x2_t data_key_lo, data_key_hi;
	4135	/* data_key_lo = (uint32x2_t) (xacc[i] & 0xFFFFFFFF);
	4136	* data_key_hi = (uint32x2_t) (xacc[i] >> 32);
	4137	* xacc[i] = UNDEFINED; */
	4138	XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi);
	4139	{ /*
	4140	* prod_hi = (data_key >> 32) * XXH_PRIME32_1;
	4141	*
	4142	* Avoid vmul_u32 + vshll_n_u32 since Clang 6 and 7 will
	4143	* incorrectly "optimize" this:
	4144	* tmp = vmul_u32(vmovn_u64(a), vmovn_u64(b));
	4145	* shifted = vshll_n_u32(tmp, 32);
	4146	* to this:
	4147	* tmp = "vmulq_u64"(a, b); // no such thing!
	4148	* shifted = vshlq_n_u64(tmp, 32);
	4149	*
	4150	* However, unlike SSE, Clang lacks a 64-bit multiply routine
	4151	* for NEON, and it scalarizes two 64-bit multiplies instead.
	4152	*
	4153	* vmull_u32 has the same timing as vmul_u32, and it avoids
	4154	* this bug completely.
	4155	* See https://bugs.llvm.org/show_bug.cgi?id=39967
	4156	*/
	4157	uint64x2_t prod_hi = vmull_u32 (data_key_hi, prime);
	4158	/* xacc[i] = prod_hi << 32; */
	4159	xacc[i] = vshlq_n_u64(prod_hi, 32);
	4160	/* xacc[i] += (prod_hi & 0xFFFFFFFF) * XXH_PRIME32_1; */
	4161	xacc[i] = vmlal_u32(xacc[i], data_key_lo, prime);
	4162	}
	4163	}
	4164	/* Scalar for the remainder. This may be a zero iteration loop. */
	4165	for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) {
	4166	XXH3_scalarScrambleRound(acc, secret, i);
	4167	}
	4168	}
	4169	}
	4170
	4171	#endif
	4172
	4173	#if (XXH_VECTOR == XXH_VSX)
	4174
	4175	XXH_FORCE_INLINE void
	4176	XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc,
	4177	const void* XXH_RESTRICT input,
	4178	const void* XXH_RESTRICT secret)
	4179	{
	4180	/* presumed aligned */
	4181	unsigned int* const xacc = (unsigned int*) acc;
	4182	xxh_u64x2 const* const xinput = (xxh_u64x2 const) input; / no alignment restriction */
	4183	xxh_u64x2 const* const xsecret = (xxh_u64x2 const) secret; / no alignment restriction */
	4184	xxh_u64x2 const v32 = { 32, 32 };
	4185	size_t i;
	4186	for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) {
	4187	/* data_vec = xinput[i]; */
	4188	xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + i);
	4189	/* key_vec = xsecret[i]; */
	4190	xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i);
	4191	xxh_u64x2 const data_key = data_vec ^ key_vec;
	4192	/* shuffled = (data_key << 32) \| (data_key >> 32); */
	4193	xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32);
	4194	/* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */
	4195	xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled);
	4196	/* acc_vec = xacc[i]; */
	4197	xxh_u64x2 acc_vec = (xxh_u64x2)vec_xl(0, xacc + 4 * i);
	4198	acc_vec += product;
	4199
	4200	/* swap high and low halves */
	4201	#ifdef __s390x__
	4202	acc_vec += vec_permi(data_vec, data_vec, 2);
	4203	#else
	4204	acc_vec += vec_xxpermdi(data_vec, data_vec, 2);
	4205	#endif
	4206	/* xacc[i] = acc_vec; */
	4207	vec_xst((xxh_u32x4)acc_vec, 0, xacc + 4 * i);
	4208	}
	4209	}
	4210
	4211	XXH_FORCE_INLINE void
	4212	XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
	4213	{
	4214	XXH_ASSERT((((size_t)acc) & 15) == 0);
	4215
	4216	{ xxh_u64x2* const xacc = (xxh_u64x2*) acc;
	4217	const xxh_u64x2* const xsecret = (const xxh_u64x2*) secret;
	4218	/* constants */
	4219	xxh_u64x2 const v32 = { 32, 32 };
	4220	xxh_u64x2 const v47 = { 47, 47 };
	4221	xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 };
	4222	size_t i;
	4223	for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) {
	4224	/* xacc[i] ^= (xacc[i] >> 47); */
	4225	xxh_u64x2 const acc_vec = xacc[i];
	4226	xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47);
	4227
	4228	/* xacc[i] ^= xsecret[i]; */
	4229	xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i);
	4230	xxh_u64x2 const data_key = data_vec ^ key_vec;
	4231
	4232	/* xacc[i] = XXH_PRIME32_1 /
	4233	/* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */
	4234	xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime);
	4235	/* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */
	4236	xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime);
	4237	xacc[i] = prod_odd + (prod_even << v32);
	4238	} }
	4239	}
	4240
	4241	#endif
	4242
	4243	/* scalar variants - universal */
	4244
	4245	/*!
	4246	* @internal
	4247	* @brief Scalar round for @ref XXH3_accumulate_512_scalar().
	4248	*
	4249	* This is extracted to its own function because the NEON path uses a combination
	4250	* of NEON and scalar.
	4251	*/
	4252	XXH_FORCE_INLINE void
	4253	XXH3_scalarRound(void* XXH_RESTRICT acc,
	4254	void const* XXH_RESTRICT input,
	4255	void const* XXH_RESTRICT secret,
	4256	size_t lane)
	4257	{
	4258	xxh_u64* xacc = (xxh_u64*) acc;
	4259	xxh_u8 const* xinput = (xxh_u8 const*) input;
	4260	xxh_u8 const* xsecret = (xxh_u8 const*) secret;
	4261	XXH_ASSERT(lane < XXH_ACC_NB);
	4262	XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0);
	4263	{
	4264	xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8);
	4265	xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8);
	4266	xacc[lane ^ 1] += data_val; /* swap adjacent lanes */
	4267	xacc[lane] += XXH_mult32to64(data_key & 0xFFFFFFFF, data_key >> 32);
	4268	}
	4269	}
	4270
	4271	/*!
	4272	* @internal
	4273	* @brief Processes a 64 byte block of data using the scalar path.
	4274	*/
	4275	XXH_FORCE_INLINE void
	4276	XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc,
	4277	const void* XXH_RESTRICT input,
	4278	const void* XXH_RESTRICT secret)
	4279	{
	4280	size_t i;
	4281	for (i=0; i < XXH_ACC_NB; i++) {
	4282	XXH3_scalarRound(acc, input, secret, i);
	4283	}
	4284	}
	4285
	4286	/*!
	4287	* @internal
	4288	* @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar().
	4289	*
	4290	* This is extracted to its own function because the NEON path uses a combination
	4291	* of NEON and scalar.
	4292	*/
	4293	XXH_FORCE_INLINE void
	4294	XXH3_scalarScrambleRound(void* XXH_RESTRICT acc,
	4295	void const* XXH_RESTRICT secret,
	4296	size_t lane)
	4297	{
	4298	xxh_u64* const xacc = (xxh_u64) acc; / presumed aligned */
	4299	const xxh_u8* const xsecret = (const xxh_u8) secret; / no alignment restriction */
	4300	XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0);
	4301	XXH_ASSERT(lane < XXH_ACC_NB);
	4302	{
	4303	xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8);
	4304	xxh_u64 acc64 = xacc[lane];
	4305	acc64 = XXH_xorshift64(acc64, 47);
	4306	acc64 ^= key64;
	4307	acc64 *= XXH_PRIME32_1;
	4308	xacc[lane] = acc64;
	4309	}
	4310	}
	4311
	4312	/*!
	4313	* @internal
	4314	* @brief Scrambles the accumulators after a large chunk has been read
	4315	*/
	4316	XXH_FORCE_INLINE void
	4317	XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
	4318	{
	4319	size_t i;
	4320	for (i=0; i < XXH_ACC_NB; i++) {
	4321	XXH3_scalarScrambleRound(acc, secret, i);
	4322	}
	4323	}
	4324
	4325	XXH_FORCE_INLINE void
	4326	XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
	4327	{
	4328	/*
	4329	* We need a separate pointer for the hack below,
	4330	* which requires a non-const pointer.
	4331	* Any decent compiler will optimize this out otherwise.
	4332	*/
	4333	const xxh_u8* kSecretPtr = XXH3_kSecret;
	4334	XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);
	4335
	4336	#if defined(__clang__) && defined(__aarch64__)
	4337	/*
	4338	* UGLY HACK:
	4339	* Clang generates a bunch of MOV/MOVK pairs for aarch64, and they are
	4340	* placed sequentially, in order, at the top of the unrolled loop.
	4341	*
	4342	* While MOVK is great for generating constants (2 cycles for a 64-bit
	4343	* constant compared to 4 cycles for LDR), it fights for bandwidth with
	4344	* the arithmetic instructions.
	4345	*
	4346	* I L S
	4347	* MOVK
	4348	* MOVK
	4349	* MOVK
	4350	* MOVK
	4351	* ADD
	4352	* SUB STR
	4353	* STR
	4354	* By forcing loads from memory (as the asm line causes Clang to assume
	4355	* that XXH3_kSecretPtr has been changed), the pipelines are used more
	4356	* efficiently:
	4357	* I L S
	4358	* LDR
	4359	* ADD LDR
	4360	* SUB STR
	4361	* STR
	4362	*
	4363	* See XXH3_NEON_LANES for details on the pipsline.
	4364	*
	4365	* XXH3_64bits_withSeed, len == 256, Snapdragon 835
	4366	* without hack: 2654.4 MB/s
	4367	* with hack: 3202.9 MB/s
	4368	*/
	4369	XXH_COMPILER_GUARD(kSecretPtr);
	4370	#endif
	4371	/*
	4372	* Note: in debug mode, this overrides the asm optimization
	4373	* and Clang will emit MOVK chains again.
	4374	*/
	4375	XXH_ASSERT(kSecretPtr == XXH3_kSecret);
	4376
	4377	{ int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16;
	4378	int i;
	4379	for (i=0; i < nbRounds; i++) {
	4380	/*
	4381	* The asm hack causes Clang to assume that kSecretPtr aliases with
	4382	* customSecret, and on aarch64, this prevented LDP from merging two
	4383	* loads together for free. Putting the loads together before the stores
	4384	* properly generates LDP.
	4385	*/
	4386	xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64;
	4387	xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64;
	4388	XXH_writeLE64((xxh_u8)customSecret + 16i, lo);
	4389	XXH_writeLE64((xxh_u8)customSecret + 16i + 8, hi);
	4390	} }
	4391	}
	4392
	4393
	4394	typedef void (XXH3_f_accumulate_512)(void XXH_RESTRICT, const void, const void);
	4395	typedef void (XXH3_f_scrambleAcc)(void XXH_RESTRICT, const void*);
	4396	typedef void (XXH3_f_initCustomSecret)(void XXH_RESTRICT, xxh_u64);
	4397
	4398
	4399	#if (XXH_VECTOR == XXH_AVX512)
	4400
	4401	#define XXH3_accumulate_512 XXH3_accumulate_512_avx512
	4402	#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512
	4403	#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512
	4404
	4405	#elif (XXH_VECTOR == XXH_AVX2)
	4406
	4407	#define XXH3_accumulate_512 XXH3_accumulate_512_avx2
	4408	#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2
	4409	#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2
	4410
	4411	#elif (XXH_VECTOR == XXH_SSE2)
	4412
	4413	#define XXH3_accumulate_512 XXH3_accumulate_512_sse2
	4414	#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2
	4415	#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2
	4416
	4417	#elif (XXH_VECTOR == XXH_NEON)
	4418
	4419	#define XXH3_accumulate_512 XXH3_accumulate_512_neon
	4420	#define XXH3_scrambleAcc XXH3_scrambleAcc_neon
	4421	#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
	4422
	4423	#elif (XXH_VECTOR == XXH_VSX)
	4424
	4425	#define XXH3_accumulate_512 XXH3_accumulate_512_vsx
	4426	#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx
	4427	#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
	4428
	4429	#else /* scalar */
	4430
	4431	#define XXH3_accumulate_512 XXH3_accumulate_512_scalar
	4432	#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar
	4433	#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
	4434
	4435	#endif
	4436
	4437
	4438
	4439	#ifndef XXH_PREFETCH_DIST
	4440	# ifdef __clang__
	4441	# define XXH_PREFETCH_DIST 320
	4442	# else
	4443	# if (XXH_VECTOR == XXH_AVX512)
	4444	# define XXH_PREFETCH_DIST 512
	4445	# else
	4446	# define XXH_PREFETCH_DIST 384
	4447	# endif
	4448	# endif /* __clang__ */
	4449	#endif /* XXH_PREFETCH_DIST */
	4450
	4451	/*
	4452	* XXH3_accumulate()
	4453	* Loops over XXH3_accumulate_512().
	4454	* Assumption: nbStripes will not overflow the secret size
	4455	*/
	4456	XXH_FORCE_INLINE void
	4457	XXH3_accumulate( xxh_u64* XXH_RESTRICT acc,
	4458	const xxh_u8* XXH_RESTRICT input,
	4459	const xxh_u8* XXH_RESTRICT secret,
	4460	size_t nbStripes,
	4461	XXH3_f_accumulate_512 f_acc512)
	4462	{
	4463	size_t n;
	4464	for (n = 0; n < nbStripes; n++ ) {
	4465	const xxh_u8* const in = input + n*XXH_STRIPE_LEN;
	4466	XXH_PREFETCH(in + XXH_PREFETCH_DIST);
	4467	f_acc512(acc,
	4468	in,
	4469	secret + n*XXH_SECRET_CONSUME_RATE);
	4470	}
	4471	}
	4472
	4473	XXH_FORCE_INLINE void
	4474	XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc,
	4475	const xxh_u8* XXH_RESTRICT input, size_t len,
	4476	const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
	4477	XXH3_f_accumulate_512 f_acc512,
	4478	XXH3_f_scrambleAcc f_scramble)
	4479	{
	4480	size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE;
	4481	size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock;
	4482	size_t const nb_blocks = (len - 1) / block_len;
	4483
	4484	size_t n;
	4485
	4486	XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
	4487
	4488	for (n = 0; n < nb_blocks; n++) {
	4489	XXH3_accumulate(acc, input + n*block_len, secret, nbStripesPerBlock, f_acc512);
	4490	f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN);
	4491	}
	4492
	4493	/* last partial block */
	4494	XXH_ASSERT(len > XXH_STRIPE_LEN);
	4495	{ size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN;
	4496	XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE));
	4497	XXH3_accumulate(acc, input + nb_blocks*block_len, secret, nbStripes, f_acc512);
	4498
	4499	/* last stripe */
	4500	{ const xxh_u8* const p = input + len - XXH_STRIPE_LEN;
	4501	#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */
	4502	f_acc512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START);
	4503	} }
	4504	}
	4505
	4506	XXH_FORCE_INLINE xxh_u64
	4507	XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret)
	4508	{
	4509	return XXH3_mul128_fold64(
	4510	acc[0] ^ XXH_readLE64(secret),
	4511	acc[1] ^ XXH_readLE64(secret+8) );
	4512	}
	4513
	4514	static XXH64_hash_t
	4515	XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start)
	4516	{
	4517	xxh_u64 result64 = start;
	4518	size_t i = 0;
	4519
	4520	for (i = 0; i < 4; i++) {
	4521	result64 += XXH3_mix2Accs(acc+2i, secret + 16i);
	4522	#if defined(__clang__) /* Clang */ \
	4523	&& (defined(__arm__) \|\| defined(__thumb__)) /* ARMv7 */ \
	4524	&& (defined(__ARM_NEON) \|\| defined(__ARM_NEON__)) /* NEON */ \
	4525	&& !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */
	4526	/*
	4527	* UGLY HACK:
	4528	* Prevent autovectorization on Clang ARMv7-a. Exact same problem as
	4529	* the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b.
	4530	* XXH3_64bits, len == 256, Snapdragon 835:
	4531	* without hack: 2063.7 MB/s
	4532	* with hack: 2560.7 MB/s
	4533	*/
	4534	XXH_COMPILER_GUARD(result64);
	4535	#endif
	4536	}
	4537
	4538	return XXH3_avalanche(result64);
	4539	}
	4540
	4541	#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \
	4542	XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 }
	4543
	4544	XXH_FORCE_INLINE XXH64_hash_t
	4545	XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len,
	4546	const void* XXH_RESTRICT secret, size_t secretSize,
	4547	XXH3_f_accumulate_512 f_acc512,
	4548	XXH3_f_scrambleAcc f_scramble)
	4549	{
	4550	XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC;
	4551
	4552	XXH3_hashLong_internal_loop(acc, (const xxh_u8)input, len, (const xxh_u8)secret, secretSize, f_acc512, f_scramble);
	4553
	4554	/* converge into final hash */
	4555	XXH_STATIC_ASSERT(sizeof(acc) == 64);
	4556	/* do not align on 8, so that the secret is different from the accumulator */
	4557	#define XXH_SECRET_MERGEACCS_START 11
	4558	XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
	4559	return XXH3_mergeAccs(acc, (const xxh_u8)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len XXH_PRIME64_1);
	4560	}
	4561
	4562	/*
	4563	* It's important for performance to transmit secret's size (when it's static)
	4564	* so that the compiler can properly optimize the vectorized loop.
	4565	* This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set.
	4566	*/
	4567	XXH_FORCE_INLINE XXH64_hash_t
	4568	XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len,
	4569	XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
	4570	{
	4571	(void)seed64;
	4572	return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate_512, XXH3_scrambleAcc);
	4573	}
	4574
	4575	/*
	4576	* It's preferable for performance that XXH3_hashLong is not inlined,
	4577	* as it results in a smaller function for small data, easier to the instruction cache.
	4578	* Note that inside this no_inline function, we do inline the internal loop,
	4579	* and provide a statically defined secret size to allow optimization of vector loop.
	4580	*/
	4581	XXH_NO_INLINE XXH64_hash_t
	4582	XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len,
	4583	XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
	4584	{
	4585	(void)seed64; (void)secret; (void)secretLen;
	4586	return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate_512, XXH3_scrambleAcc);
	4587	}
	4588
	4589	/*
	4590	* XXH3_hashLong_64b_withSeed():
	4591	* Generate a custom key based on alteration of default XXH3_kSecret with the seed,
	4592	* and then use this key for long mode hashing.
	4593	*
	4594	* This operation is decently fast but nonetheless costs a little bit of time.
	4595	* Try to avoid it whenever possible (typically when seed==0).
	4596	*
	4597	* It's important for performance that XXH3_hashLong is not inlined. Not sure
	4598	* why (uop cache maybe?), but the difference is large and easily measurable.
	4599	*/
	4600	XXH_FORCE_INLINE XXH64_hash_t
	4601	XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len,
	4602	XXH64_hash_t seed,
	4603	XXH3_f_accumulate_512 f_acc512,
	4604	XXH3_f_scrambleAcc f_scramble,
	4605	XXH3_f_initCustomSecret f_initSec)
	4606	{
	4607	if (seed == 0)
	4608	return XXH3_hashLong_64b_internal(input, len,
	4609	XXH3_kSecret, sizeof(XXH3_kSecret),
	4610	f_acc512, f_scramble);
	4611	{ XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
	4612	f_initSec(secret, seed);
	4613	return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret),
	4614	f_acc512, f_scramble);
	4615	}
	4616	}
	4617
	4618	/*
	4619	* It's important for performance that XXH3_hashLong is not inlined.
	4620	*/
	4621	XXH_NO_INLINE XXH64_hash_t
	4622	XXH3_hashLong_64b_withSeed(const void* input, size_t len,
	4623	XXH64_hash_t seed, const xxh_u8* secret, size_t secretLen)
	4624	{
	4625	(void)secret; (void)secretLen;
	4626	return XXH3_hashLong_64b_withSeed_internal(input, len, seed,
	4627	XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret);
	4628	}
	4629
	4630
	4631	typedef XXH64_hash_t (XXH3_hashLong64_f)(const void XXH_RESTRICT, size_t,
	4632	XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t);
	4633
	4634	XXH_FORCE_INLINE XXH64_hash_t
	4635	XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len,
	4636	XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen,
	4637	XXH3_hashLong64_f f_hashLong)
	4638	{
	4639	XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN);
	4640	/*
	4641	* If an action is to be taken if `secretLen` condition is not respected,
	4642	* it should be done here.
	4643	* For now, it's a contract pre-condition.
	4644	* Adding a check and a branch here would cost performance at every hash.
	4645	* Also, note that function signature doesn't offer room to return an error.
	4646	*/
	4647	if (len <= 16)
	4648	return XXH3_len_0to16_64b((const xxh_u8)input, len, (const xxh_u8)secret, seed64);
	4649	if (len <= 128)
	4650	return XXH3_len_17to128_64b((const xxh_u8)input, len, (const xxh_u8)secret, secretLen, seed64);
	4651	if (len <= XXH3_MIDSIZE_MAX)
	4652	return XXH3_len_129to240_64b((const xxh_u8)input, len, (const xxh_u8)secret, secretLen, seed64);
	4653	return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen);
	4654	}
	4655
	4656
	4657	/* === Public entry point === */
	4658
	4659	/! @ingroup xxh3_family /
	4660	XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* input, size_t len)
	4661	{
	4662	return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default);
	4663	}
	4664
	4665	/! @ingroup xxh3_family /
	4666	XXH_PUBLIC_API XXH64_hash_t
	4667	XXH3_64bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize)
	4668	{
	4669	return XXH3_64bits_internal(input, len, 0, secret, secretSize, XXH3_hashLong_64b_withSecret);
	4670	}
	4671
	4672	/! @ingroup xxh3_family /
	4673	XXH_PUBLIC_API XXH64_hash_t
	4674	XXH3_64bits_withSeed(const void* input, size_t len, XXH64_hash_t seed)
	4675	{
	4676	return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed);
	4677	}
	4678
	4679	XXH_PUBLIC_API XXH64_hash_t
	4680	XXH3_64bits_withSecretandSeed(const void* input, size_t len, const void* secret, size_t secretSize, XXH64_hash_t seed)
	4681	{
	4682	if (len <= XXH3_MIDSIZE_MAX)
	4683	return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL);
	4684	return XXH3_hashLong_64b_withSecret(input, len, seed, (const xxh_u8*)secret, secretSize);
	4685	}
	4686
	4687
	4688	/* === XXH3 streaming === */
	4689
	4690	/*
	4691	* Malloc's a pointer that is always aligned to align.
	4692	*
	4693	* This must be freed with `XXH_alignedFree()`.
	4694	*
	4695	* malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte
	4696	* alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2
	4697	* or on 32-bit, the 16 byte aligned loads in SSE2 and NEON.
	4698	*
	4699	* This underalignment previously caused a rather obvious crash which went
	4700	* completely unnoticed due to XXH3_createState() not actually being tested.
	4701	* Credit to RedSpah for noticing this bug.
	4702	*
	4703	* The alignment is done manually: Functions like posix_memalign or _mm_malloc
	4704	* are avoided: To maintain portability, we would have to write a fallback
	4705	* like this anyways, and besides, testing for the existence of library
	4706	* functions without relying on external build tools is impossible.
	4707	*
	4708	* The method is simple: Overallocate, manually align, and store the offset
	4709	* to the original behind the returned pointer.
	4710	*
	4711	* Align must be a power of 2 and 8 <= align <= 128.
	4712	*/
	4713	static void* XXH_alignedMalloc(size_t s, size_t align)
	4714	{
	4715	XXH_ASSERT(align <= 128 && align >= 8); /* range check */
	4716	XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */
	4717	XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */
	4718	{ /* Overallocate to make room for manual realignment and an offset byte */
	4719	xxh_u8* base = (xxh_u8*)XXH_malloc(s + align);
	4720	if (base != NULL) {
	4721	/*
	4722	* Get the offset needed to align this pointer.
	4723	*
	4724	* Even if the returned pointer is aligned, there will always be
	4725	* at least one byte to store the offset to the original pointer.
	4726	*/
	4727	size_t offset = align - ((size_t)base & (align - 1)); /* base % align */
	4728	/* Add the offset for the now-aligned pointer */
	4729	xxh_u8* ptr = base + offset;
	4730
	4731	XXH_ASSERT((size_t)ptr % align == 0);
	4732
	4733	/* Store the offset immediately before the returned pointer. */
	4734	ptr[-1] = (xxh_u8)offset;
	4735	return ptr;
	4736	}
	4737	return NULL;
	4738	}
	4739	}
	4740	/*
	4741	* Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass
	4742	* normal malloc'd pointers, XXH_alignedMalloc has a specific data layout.
	4743	*/
	4744	static void XXH_alignedFree(void* p)
	4745	{
	4746	if (p != NULL) {
	4747	xxh_u8* ptr = (xxh_u8*)p;
	4748	/* Get the offset byte we added in XXH_malloc. */
	4749	xxh_u8 offset = ptr[-1];
	4750	/* Free the original malloc'd pointer */
	4751	xxh_u8* base = ptr - offset;
	4752	XXH_free(base);
	4753	}
	4754	}
	4755	/! @ingroup xxh3_family /
	4756	XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void)
	4757	{
	4758	XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64);
	4759	if (state==NULL) return NULL;
	4760	XXH3_INITSTATE(state);
	4761	return state;
	4762	}
	4763
	4764	/! @ingroup xxh3_family /
	4765	XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr)
	4766	{
	4767	XXH_alignedFree(statePtr);
	4768	return XXH_OK;
	4769	}
	4770
	4771	/! @ingroup xxh3_family /
	4772	XXH_PUBLIC_API void
	4773	XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state)
	4774	{
	4775	XXH_memcpy(dst_state, src_state, sizeof(*dst_state));
	4776	}
	4777
	4778	static void
	4779	XXH3_reset_internal(XXH3_state_t* statePtr,
	4780	XXH64_hash_t seed,
	4781	const void* secret, size_t secretSize)
	4782	{
	4783	size_t const initStart = offsetof(XXH3_state_t, bufferedSize);
	4784	size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart;
	4785	XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart);
	4786	XXH_ASSERT(statePtr != NULL);
	4787	/* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */
	4788	memset((char*)statePtr + initStart, 0, initLength);
	4789	statePtr->acc[0] = XXH_PRIME32_3;
	4790	statePtr->acc[1] = XXH_PRIME64_1;
	4791	statePtr->acc[2] = XXH_PRIME64_2;
	4792	statePtr->acc[3] = XXH_PRIME64_3;
	4793	statePtr->acc[4] = XXH_PRIME64_4;
	4794	statePtr->acc[5] = XXH_PRIME32_2;
	4795	statePtr->acc[6] = XXH_PRIME64_5;
	4796	statePtr->acc[7] = XXH_PRIME32_1;
	4797	statePtr->seed = seed;
	4798	statePtr->useSeed = (seed != 0);
	4799	statePtr->extSecret = (const unsigned char*)secret;
	4800	XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
	4801	statePtr->secretLimit = secretSize - XXH_STRIPE_LEN;
	4802	statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE;
	4803	}
	4804
	4805	/! @ingroup xxh3_family /
	4806	XXH_PUBLIC_API XXH_errorcode
	4807	XXH3_64bits_reset(XXH3_state_t* statePtr)
	4808	{
	4809	if (statePtr == NULL) return XXH_ERROR;
	4810	XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE);
	4811	return XXH_OK;
	4812	}
	4813
	4814	/! @ingroup xxh3_family /
	4815	XXH_PUBLIC_API XXH_errorcode
	4816	XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize)
	4817	{
	4818	if (statePtr == NULL) return XXH_ERROR;
	4819	XXH3_reset_internal(statePtr, 0, secret, secretSize);
	4820	if (secret == NULL) return XXH_ERROR;
	4821	if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
	4822	return XXH_OK;
	4823	}
	4824
	4825	/! @ingroup xxh3_family /
	4826	XXH_PUBLIC_API XXH_errorcode
	4827	XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed)
	4828	{
	4829	if (statePtr == NULL) return XXH_ERROR;
	4830	if (seed==0) return XXH3_64bits_reset(statePtr);
	4831	if ((seed != statePtr->seed) \|\| (statePtr->extSecret != NULL))
	4832	XXH3_initCustomSecret(statePtr->customSecret, seed);
	4833	XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE);
	4834	return XXH_OK;
	4835	}
	4836
	4837	/! @ingroup xxh3_family /
	4838	XXH_PUBLIC_API XXH_errorcode
	4839	XXH3_64bits_reset_withSecretandSeed(XXH3_state_t* statePtr, const void* secret, size_t secretSize, XXH64_hash_t seed64)
	4840	{
	4841	if (statePtr == NULL) return XXH_ERROR;
	4842	if (secret == NULL) return XXH_ERROR;
	4843	if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
	4844	XXH3_reset_internal(statePtr, seed64, secret, secretSize);
	4845	statePtr->useSeed = 1; /* always, even if seed64==0 */
	4846	return XXH_OK;
	4847	}
	4848
	4849	/* Note : when XXH3_consumeStripes() is invoked,
	4850	* there must be a guarantee that at least one more byte must be consumed from input
	4851	* so that the function can blindly consume all stripes using the "normal" secret segment */
	4852	XXH_FORCE_INLINE void
	4853	XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc,
	4854	size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock,
	4855	const xxh_u8* XXH_RESTRICT input, size_t nbStripes,
	4856	const xxh_u8* XXH_RESTRICT secret, size_t secretLimit,
	4857	XXH3_f_accumulate_512 f_acc512,
	4858	XXH3_f_scrambleAcc f_scramble)
	4859	{
	4860	XXH_ASSERT(nbStripes <= nbStripesPerBlock); /* can handle max 1 scramble per invocation */
	4861	XXH_ASSERT(*nbStripesSoFarPtr < nbStripesPerBlock);
	4862	if (nbStripesPerBlock - *nbStripesSoFarPtr <= nbStripes) {
	4863	/* need a scrambling operation */
	4864	size_t const nbStripesToEndofBlock = nbStripesPerBlock - *nbStripesSoFarPtr;
	4865	size_t const nbStripesAfterBlock = nbStripes - nbStripesToEndofBlock;
	4866	XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripesToEndofBlock, f_acc512);
	4867	f_scramble(acc, secret + secretLimit);
	4868	XXH3_accumulate(acc, input + nbStripesToEndofBlock * XXH_STRIPE_LEN, secret, nbStripesAfterBlock, f_acc512);
	4869	*nbStripesSoFarPtr = nbStripesAfterBlock;
	4870	} else {
	4871	XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripes, f_acc512);
	4872	*nbStripesSoFarPtr += nbStripes;
	4873	}
	4874	}
	4875
	4876	#ifndef XXH3_STREAM_USE_STACK
	4877	# ifndef __clang__ /* clang doesn't need additional stack space */
	4878	# define XXH3_STREAM_USE_STACK 1
	4879	# endif
	4880	#endif
	4881	/*
	4882	* Both XXH3_64bits_update and XXH3_128bits_update use this routine.
	4883	*/
	4884	XXH_FORCE_INLINE XXH_errorcode
	4885	XXH3_update(XXH3_state_t* XXH_RESTRICT const state,
	4886	const xxh_u8* XXH_RESTRICT input, size_t len,
	4887	XXH3_f_accumulate_512 f_acc512,
	4888	XXH3_f_scrambleAcc f_scramble)
	4889	{
	4890	if (input==NULL) {
	4891	XXH_ASSERT(len == 0);
	4892	return XXH_OK;
	4893	}
	4894
	4895	XXH_ASSERT(state != NULL);
	4896	{ const xxh_u8* const bEnd = input + len;
	4897	const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
	4898	#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1
	4899	/* For some reason, gcc and MSVC seem to suffer greatly
	4900	* when operating accumulators directly into state.
	4901	* Operating into stack space seems to enable proper optimization.
	4902	* clang, on the other hand, doesn't seem to need this trick */
	4903	XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; memcpy(acc, state->acc, sizeof(acc));
	4904	#else
	4905	xxh_u64* XXH_RESTRICT const acc = state->acc;
	4906	#endif
	4907	state->totalLen += len;
	4908	XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE);
	4909
	4910	/* small input : just fill in tmp buffer */
	4911	if (state->bufferedSize + len <= XXH3_INTERNALBUFFER_SIZE) {
	4912	XXH_memcpy(state->buffer + state->bufferedSize, input, len);
	4913	state->bufferedSize += (XXH32_hash_t)len;
	4914	return XXH_OK;
	4915	}
	4916
	4917	/* total input is now > XXH3_INTERNALBUFFER_SIZE */
	4918	#define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN)
	4919	XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */
	4920
	4921	/*
	4922	* Internal buffer is partially filled (always, except at beginning)
	4923	* Complete it, then consume it.
	4924	*/
	4925	if (state->bufferedSize) {
	4926	size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize;
	4927	XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize);
	4928	input += loadSize;
	4929	XXH3_consumeStripes(acc,
	4930	&state->nbStripesSoFar, state->nbStripesPerBlock,
	4931	state->buffer, XXH3_INTERNALBUFFER_STRIPES,
	4932	secret, state->secretLimit,
	4933	f_acc512, f_scramble);
	4934	state->bufferedSize = 0;
	4935	}
	4936	XXH_ASSERT(input < bEnd);
	4937
	4938	/* large input to consume : ingest per full block */
	4939	if ((size_t)(bEnd - input) > state->nbStripesPerBlock * XXH_STRIPE_LEN) {
	4940	size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN;
	4941	XXH_ASSERT(state->nbStripesPerBlock >= state->nbStripesSoFar);
	4942	/* join to current block's end */
	4943	{ size_t const nbStripesToEnd = state->nbStripesPerBlock - state->nbStripesSoFar;
	4944	XXH_ASSERT(nbStripesToEnd <= nbStripes);
	4945	XXH3_accumulate(acc, input, secret + state->nbStripesSoFar * XXH_SECRET_CONSUME_RATE, nbStripesToEnd, f_acc512);
	4946	f_scramble(acc, secret + state->secretLimit);
	4947	state->nbStripesSoFar = 0;
	4948	input += nbStripesToEnd * XXH_STRIPE_LEN;
	4949	nbStripes -= nbStripesToEnd;
	4950	}
	4951	/* consume per entire blocks */
	4952	while(nbStripes >= state->nbStripesPerBlock) {
	4953	XXH3_accumulate(acc, input, secret, state->nbStripesPerBlock, f_acc512);
	4954	f_scramble(acc, secret + state->secretLimit);
	4955	input += state->nbStripesPerBlock * XXH_STRIPE_LEN;
	4956	nbStripes -= state->nbStripesPerBlock;
	4957	}
	4958	/* consume last partial block */
	4959	XXH3_accumulate(acc, input, secret, nbStripes, f_acc512);
	4960	input += nbStripes * XXH_STRIPE_LEN;
	4961	XXH_ASSERT(input < bEnd); /* at least some bytes left */
	4962	state->nbStripesSoFar = nbStripes;
	4963	/* buffer predecessor of last partial stripe */
	4964	XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN);
	4965	XXH_ASSERT(bEnd - input <= XXH_STRIPE_LEN);
	4966	} else {
	4967	/* content to consume <= block size */
	4968	/* Consume input by a multiple of internal buffer size */
	4969	if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) {
	4970	const xxh_u8* const limit = bEnd - XXH3_INTERNALBUFFER_SIZE;
	4971	do {
	4972	XXH3_consumeStripes(acc,
	4973	&state->nbStripesSoFar, state->nbStripesPerBlock,
	4974	input, XXH3_INTERNALBUFFER_STRIPES,
	4975	secret, state->secretLimit,
	4976	f_acc512, f_scramble);
	4977	input += XXH3_INTERNALBUFFER_SIZE;
	4978	} while (input<limit);
	4979	/* buffer predecessor of last partial stripe */
	4980	XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN);
	4981	}
	4982	}
	4983
	4984	/* Some remaining input (always) : buffer it */
	4985	XXH_ASSERT(input < bEnd);
	4986	XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE);
	4987	XXH_ASSERT(state->bufferedSize == 0);
	4988	XXH_memcpy(state->buffer, input, (size_t)(bEnd-input));
	4989	state->bufferedSize = (XXH32_hash_t)(bEnd-input);
	4990	#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1
	4991	/* save stack accumulators into state */
	4992	memcpy(state->acc, acc, sizeof(acc));
	4993	#endif
	4994	}
	4995
	4996	return XXH_OK;
	4997	}
	4998
	4999	/! @ingroup xxh3_family /
	5000	XXH_PUBLIC_API XXH_errorcode
	5001	XXH3_64bits_update(XXH3_state_t* state, const void* input, size_t len)
	5002	{
	5003	return XXH3_update(state, (const xxh_u8*)input, len,
	5004	XXH3_accumulate_512, XXH3_scrambleAcc);
	5005	}
	5006
	5007
	5008	XXH_FORCE_INLINE void
	5009	XXH3_digest_long (XXH64_hash_t* acc,
	5010	const XXH3_state_t* state,
	5011	const unsigned char* secret)
	5012	{
	5013	/*
	5014	* Digest on a local copy. This way, the state remains unaltered, and it can
	5015	* continue ingesting more input afterwards.
	5016	*/
	5017	XXH_memcpy(acc, state->acc, sizeof(state->acc));
	5018	if (state->bufferedSize >= XXH_STRIPE_LEN) {
	5019	size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN;
	5020	size_t nbStripesSoFar = state->nbStripesSoFar;
	5021	XXH3_consumeStripes(acc,
	5022	&nbStripesSoFar, state->nbStripesPerBlock,
	5023	state->buffer, nbStripes,
	5024	secret, state->secretLimit,
	5025	XXH3_accumulate_512, XXH3_scrambleAcc);
	5026	/* last stripe */
	5027	XXH3_accumulate_512(acc,
	5028	state->buffer + state->bufferedSize - XXH_STRIPE_LEN,
	5029	secret + state->secretLimit - XXH_SECRET_LASTACC_START);
	5030	} else { /* bufferedSize < XXH_STRIPE_LEN */
	5031	xxh_u8 lastStripe[XXH_STRIPE_LEN];
	5032	size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize;
	5033	XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */
	5034	XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize);
	5035	XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize);
	5036	XXH3_accumulate_512(acc,
	5037	lastStripe,
	5038	secret + state->secretLimit - XXH_SECRET_LASTACC_START);
	5039	}
	5040	}
	5041
	5042	/! @ingroup xxh3_family /
	5043	XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* state)
	5044	{
	5045	const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
	5046	if (state->totalLen > XXH3_MIDSIZE_MAX) {
	5047	XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB];
	5048	XXH3_digest_long(acc, state, secret);
	5049	return XXH3_mergeAccs(acc,
	5050	secret + XXH_SECRET_MERGEACCS_START,
	5051	(xxh_u64)state->totalLen * XXH_PRIME64_1);
	5052	}
	5053	/* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */
	5054	if (state->useSeed)
	5055	return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
	5056	return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen),
	5057	secret, state->secretLimit + XXH_STRIPE_LEN);
	5058	}
	5059
	5060
	5061
	5062	/* ==========================================
	5063	* XXH3 128 bits (a.k.a XXH128)
	5064	* ==========================================
	5065	* XXH3's 128-bit variant has better mixing and strength than the 64-bit variant,
	5066	* even without counting the significantly larger output size.
	5067	*
	5068	* For example, extra steps are taken to avoid the seed-dependent collisions
	5069	* in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B).
	5070	*
	5071	* This strength naturally comes at the cost of some speed, especially on short
	5072	* lengths. Note that longer hashes are about as fast as the 64-bit version
	5073	* due to it using only a slight modification of the 64-bit loop.
	5074	*
	5075	* XXH128 is also more oriented towards 64-bit machines. It is still extremely
	5076	* fast for a _128-bit_ hash on 32-bit (it usually clears XXH64).
	5077	*/
	5078
	5079	XXH_FORCE_INLINE XXH128_hash_t
	5080	XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
	5081	{
	5082	/* A doubled version of 1to3_64b with different constants. */
	5083	XXH_ASSERT(input != NULL);
	5084	XXH_ASSERT(1 <= len && len <= 3);
	5085	XXH_ASSERT(secret != NULL);
	5086	/*
	5087	* len = 1: combinedl = { input[0], 0x01, input[0], input[0] }
	5088	* len = 2: combinedl = { input[1], 0x02, input[0], input[1] }
	5089	* len = 3: combinedl = { input[2], 0x03, input[0], input[1] }
	5090	*/
	5091	{ xxh_u8 const c1 = input[0];
	5092	xxh_u8 const c2 = input[len >> 1];
	5093	xxh_u8 const c3 = input[len - 1];
	5094	xxh_u32 const combinedl = ((xxh_u32)c1 <<16) \| ((xxh_u32)c2 << 24)
	5095	\| ((xxh_u32)c3 << 0) \| ((xxh_u32)len << 8);
	5096	xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13);
	5097	xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed;
	5098	xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed;
	5099	xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl;
	5100	xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph;
	5101	XXH128_hash_t h128;
	5102	h128.low64 = XXH64_avalanche(keyed_lo);
	5103	h128.high64 = XXH64_avalanche(keyed_hi);
	5104	return h128;
	5105	}
	5106	}
	5107
	5108	XXH_FORCE_INLINE XXH128_hash_t
	5109	XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
	5110	{
	5111	XXH_ASSERT(input != NULL);
	5112	XXH_ASSERT(secret != NULL);
	5113	XXH_ASSERT(4 <= len && len <= 8);
	5114	seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32;
	5115	{ xxh_u32 const input_lo = XXH_readLE32(input);
	5116	xxh_u32 const input_hi = XXH_readLE32(input + len - 4);
	5117	xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32);
	5118	xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed;
	5119	xxh_u64 const keyed = input_64 ^ bitflip;
	5120
	5121	/* Shift len to the left to ensure it is even, this avoids even multiplies. */
	5122	XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2));
	5123
	5124	m128.high64 += (m128.low64 << 1);
	5125	m128.low64 ^= (m128.high64 >> 3);
	5126
	5127	m128.low64 = XXH_xorshift64(m128.low64, 35);
	5128	m128.low64 *= 0x9FB21C651E98DF25ULL;
	5129	m128.low64 = XXH_xorshift64(m128.low64, 28);
	5130	m128.high64 = XXH3_avalanche(m128.high64);
	5131	return m128;
	5132	}
	5133	}
	5134
	5135	XXH_FORCE_INLINE XXH128_hash_t
	5136	XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
	5137	{
	5138	XXH_ASSERT(input != NULL);
	5139	XXH_ASSERT(secret != NULL);
	5140	XXH_ASSERT(9 <= len && len <= 16);
	5141	{ xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed;
	5142	xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed;
	5143	xxh_u64 const input_lo = XXH_readLE64(input);
	5144	xxh_u64 input_hi = XXH_readLE64(input + len - 8);
	5145	XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1);
	5146	/*
	5147	* Put len in the middle of m128 to ensure that the length gets mixed to
	5148	* both the low and high bits in the 128x64 multiply below.
	5149	*/
	5150	m128.low64 += (xxh_u64)(len - 1) << 54;
	5151	input_hi ^= bitfliph;
	5152	/*
	5153	* Add the high 32 bits of input_hi to the high 32 bits of m128, then
	5154	* add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to
	5155	* the high 64 bits of m128.
	5156	*
	5157	* The best approach to this operation is different on 32-bit and 64-bit.
	5158	*/
	5159	if (sizeof(void ) < sizeof(xxh_u64)) { / 32-bit */
	5160	/*
	5161	* 32-bit optimized version, which is more readable.
	5162	*
	5163	* On 32-bit, it removes an ADC and delays a dependency between the two
	5164	* halves of m128.high64, but it generates an extra mask on 64-bit.
	5165	*/
	5166	m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2);
	5167	} else {
	5168	/*
	5169	* 64-bit optimized (albeit more confusing) version.
	5170	*
	5171	* Uses some properties of addition and multiplication to remove the mask:
	5172	*
	5173	* Let:
	5174	* a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF)
	5175	* b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000)
	5176	* c = XXH_PRIME32_2
	5177	*
	5178	* a + (b * c)
	5179	* Inverse Property: x + y - x == y
	5180	* a + (b * (1 + c - 1))
	5181	* Distributive Property: x * (y + z) == (x * y) + (x * z)
	5182	* a + (b * 1) + (b * (c - 1))
	5183	* Identity Property: x * 1 == x
	5184	* a + b + (b * (c - 1))
	5185	*
	5186	* Substitute a, b, and c:
	5187	* input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1))
	5188	*
	5189	* Since input_hi.hi + input_hi.lo == input_hi, we get this:
	5190	* input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1))
	5191	*/
	5192	m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1);
	5193	}
	5194	/* m128 ^= XXH_swap64(m128 >> 64); */
	5195	m128.low64 ^= XXH_swap64(m128.high64);
	5196
	5197	{ /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */
	5198	XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2);
	5199	h128.high64 += m128.high64 * XXH_PRIME64_2;
	5200
	5201	h128.low64 = XXH3_avalanche(h128.low64);
	5202	h128.high64 = XXH3_avalanche(h128.high64);
	5203	return h128;
	5204	} }
	5205	}
	5206
	5207	/*
	5208	* Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN
	5209	*/
	5210	XXH_FORCE_INLINE XXH128_hash_t
	5211	XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
	5212	{
	5213	XXH_ASSERT(len <= 16);
	5214	{ if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed);
	5215	if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed);
	5216	if (len) return XXH3_len_1to3_128b(input, len, secret, seed);
	5217	{ XXH128_hash_t h128;
	5218	xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72);
	5219	xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88);
	5220	h128.low64 = XXH64_avalanche(seed ^ bitflipl);
	5221	h128.high64 = XXH64_avalanche( seed ^ bitfliph);
	5222	return h128;
	5223	} }
	5224	}
	5225
	5226	/*
	5227	* A bit slower than XXH3_mix16B, but handles multiply by zero better.
	5228	*/
	5229	XXH_FORCE_INLINE XXH128_hash_t
	5230	XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2,
	5231	const xxh_u8* secret, XXH64_hash_t seed)
	5232	{
	5233	acc.low64 += XXH3_mix16B (input_1, secret+0, seed);
	5234	acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8);
	5235	acc.high64 += XXH3_mix16B (input_2, secret+16, seed);
	5236	acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8);
	5237	return acc;
	5238	}
	5239
	5240
	5241	XXH_FORCE_INLINE XXH128_hash_t
	5242	XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len,
	5243	const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
	5244	XXH64_hash_t seed)
	5245	{
	5246	XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
	5247	XXH_ASSERT(16 < len && len <= 128);
	5248
	5249	{ XXH128_hash_t acc;
	5250	acc.low64 = len * XXH_PRIME64_1;
	5251	acc.high64 = 0;
	5252	if (len > 32) {
	5253	if (len > 64) {
	5254	if (len > 96) {
	5255	acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed);
	5256	}
	5257	acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed);
	5258	}
	5259	acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed);
	5260	}
	5261	acc = XXH128_mix32B(acc, input, input+len-16, secret, seed);
	5262	{ XXH128_hash_t h128;
	5263	h128.low64 = acc.low64 + acc.high64;
	5264	h128.high64 = (acc.low64 * XXH_PRIME64_1)
	5265	+ (acc.high64 * XXH_PRIME64_4)
	5266	+ ((len - seed) * XXH_PRIME64_2);
	5267	h128.low64 = XXH3_avalanche(h128.low64);
	5268	h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64);
	5269	return h128;
	5270	}
	5271	}
	5272	}
	5273
	5274	XXH_NO_INLINE XXH128_hash_t
	5275	XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len,
	5276	const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
	5277	XXH64_hash_t seed)
	5278	{
	5279	XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
	5280	XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
	5281
	5282	{ XXH128_hash_t acc;
	5283	int const nbRounds = (int)len / 32;
	5284	int i;
	5285	acc.low64 = len * XXH_PRIME64_1;
	5286	acc.high64 = 0;
	5287	for (i=0; i<4; i++) {
	5288	acc = XXH128_mix32B(acc,
	5289	input + (32 * i),
	5290	input + (32 * i) + 16,
	5291	secret + (32 * i),
	5292	seed);
	5293	}
	5294	acc.low64 = XXH3_avalanche(acc.low64);
	5295	acc.high64 = XXH3_avalanche(acc.high64);
	5296	XXH_ASSERT(nbRounds >= 4);
	5297	for (i=4 ; i < nbRounds; i++) {
	5298	acc = XXH128_mix32B(acc,
	5299	input + (32 * i),
	5300	input + (32 * i) + 16,
	5301	secret + XXH3_MIDSIZE_STARTOFFSET + (32 * (i - 4)),
	5302	seed);
	5303	}
	5304	/* last bytes */
	5305	acc = XXH128_mix32B(acc,
	5306	input + len - 16,
	5307	input + len - 32,
	5308	secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16,
	5309	0ULL - seed);
	5310
	5311	{ XXH128_hash_t h128;
	5312	h128.low64 = acc.low64 + acc.high64;
	5313	h128.high64 = (acc.low64 * XXH_PRIME64_1)
	5314	+ (acc.high64 * XXH_PRIME64_4)
	5315	+ ((len - seed) * XXH_PRIME64_2);
	5316	h128.low64 = XXH3_avalanche(h128.low64);
	5317	h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64);
	5318	return h128;
	5319	}
	5320	}
	5321	}
	5322
	5323	XXH_FORCE_INLINE XXH128_hash_t
	5324	XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len,
	5325	const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
	5326	XXH3_f_accumulate_512 f_acc512,
	5327	XXH3_f_scrambleAcc f_scramble)
	5328	{
	5329	XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC;
	5330
	5331	XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc512, f_scramble);
	5332
	5333	/* converge into final hash */
	5334	XXH_STATIC_ASSERT(sizeof(acc) == 64);
	5335	XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
	5336	{ XXH128_hash_t h128;
	5337	h128.low64 = XXH3_mergeAccs(acc,
	5338	secret + XXH_SECRET_MERGEACCS_START,
	5339	(xxh_u64)len * XXH_PRIME64_1);
	5340	h128.high64 = XXH3_mergeAccs(acc,
	5341	secret + secretSize
	5342	- sizeof(acc) - XXH_SECRET_MERGEACCS_START,
	5343	~((xxh_u64)len * XXH_PRIME64_2));
	5344	return h128;
	5345	}
	5346	}
	5347
	5348	/*
	5349	* It's important for performance that XXH3_hashLong is not inlined.
	5350	*/
	5351	XXH_NO_INLINE XXH128_hash_t
	5352	XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len,
	5353	XXH64_hash_t seed64,
	5354	const void* XXH_RESTRICT secret, size_t secretLen)
	5355	{
	5356	(void)seed64; (void)secret; (void)secretLen;
	5357	return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret),
	5358	XXH3_accumulate_512, XXH3_scrambleAcc);
	5359	}
	5360
	5361	/*
	5362	* It's important for performance to pass @secretLen (when it's static)
	5363	* to the compiler, so that it can properly optimize the vectorized loop.
	5364	*/
	5365	XXH_FORCE_INLINE XXH128_hash_t
	5366	XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len,
	5367	XXH64_hash_t seed64,
	5368	const void* XXH_RESTRICT secret, size_t secretLen)
	5369	{
	5370	(void)seed64;
	5371	return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen,
	5372	XXH3_accumulate_512, XXH3_scrambleAcc);
	5373	}
	5374
	5375	XXH_FORCE_INLINE XXH128_hash_t
	5376	XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len,
	5377	XXH64_hash_t seed64,
	5378	XXH3_f_accumulate_512 f_acc512,
	5379	XXH3_f_scrambleAcc f_scramble,
	5380	XXH3_f_initCustomSecret f_initSec)
	5381	{
	5382	if (seed64 == 0)
	5383	return XXH3_hashLong_128b_internal(input, len,
	5384	XXH3_kSecret, sizeof(XXH3_kSecret),
	5385	f_acc512, f_scramble);
	5386	{ XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
	5387	f_initSec(secret, seed64);
	5388	return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret),
	5389	f_acc512, f_scramble);
	5390	}
	5391	}
	5392
	5393	/*
	5394	* It's important for performance that XXH3_hashLong is not inlined.
	5395	*/
	5396	XXH_NO_INLINE XXH128_hash_t
	5397	XXH3_hashLong_128b_withSeed(const void* input, size_t len,
	5398	XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen)
	5399	{
	5400	(void)secret; (void)secretLen;
	5401	return XXH3_hashLong_128b_withSeed_internal(input, len, seed64,
	5402	XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret);
	5403	}
	5404
	5405	typedef XXH128_hash_t (XXH3_hashLong128_f)(const void XXH_RESTRICT, size_t,
	5406	XXH64_hash_t, const void* XXH_RESTRICT, size_t);
	5407
	5408	XXH_FORCE_INLINE XXH128_hash_t
	5409	XXH3_128bits_internal(const void* input, size_t len,
	5410	XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen,
	5411	XXH3_hashLong128_f f_hl128)
	5412	{
	5413	XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN);
	5414	/*
	5415	* If an action is to be taken if `secret` conditions are not respected,
	5416	* it should be done here.
	5417	* For now, it's a contract pre-condition.
	5418	* Adding a check and a branch here would cost performance at every hash.
	5419	*/
	5420	if (len <= 16)
	5421	return XXH3_len_0to16_128b((const xxh_u8)input, len, (const xxh_u8)secret, seed64);
	5422	if (len <= 128)
	5423	return XXH3_len_17to128_128b((const xxh_u8)input, len, (const xxh_u8)secret, secretLen, seed64);
	5424	if (len <= XXH3_MIDSIZE_MAX)
	5425	return XXH3_len_129to240_128b((const xxh_u8)input, len, (const xxh_u8)secret, secretLen, seed64);
	5426	return f_hl128(input, len, seed64, secret, secretLen);
	5427	}
	5428
	5429
	5430	/* === Public XXH128 API === */
	5431
	5432	/! @ingroup xxh3_family /
	5433	XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* input, size_t len)
	5434	{
	5435	return XXH3_128bits_internal(input, len, 0,
	5436	XXH3_kSecret, sizeof(XXH3_kSecret),
	5437	XXH3_hashLong_128b_default);
	5438	}
	5439
	5440	/! @ingroup xxh3_family /
	5441	XXH_PUBLIC_API XXH128_hash_t
	5442	XXH3_128bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize)
	5443	{
	5444	return XXH3_128bits_internal(input, len, 0,
	5445	(const xxh_u8*)secret, secretSize,
	5446	XXH3_hashLong_128b_withSecret);
	5447	}
	5448
	5449	/! @ingroup xxh3_family /
	5450	XXH_PUBLIC_API XXH128_hash_t
	5451	XXH3_128bits_withSeed(const void* input, size_t len, XXH64_hash_t seed)
	5452	{
	5453	return XXH3_128bits_internal(input, len, seed,
	5454	XXH3_kSecret, sizeof(XXH3_kSecret),
	5455	XXH3_hashLong_128b_withSeed);
	5456	}
	5457
	5458	/! @ingroup xxh3_family /
	5459	XXH_PUBLIC_API XXH128_hash_t
	5460	XXH3_128bits_withSecretandSeed(const void* input, size_t len, const void* secret, size_t secretSize, XXH64_hash_t seed)
	5461	{
	5462	if (len <= XXH3_MIDSIZE_MAX)
	5463	return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL);
	5464	return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize);
	5465	}
	5466
	5467	/! @ingroup xxh3_family /
	5468	XXH_PUBLIC_API XXH128_hash_t
	5469	XXH128(const void* input, size_t len, XXH64_hash_t seed)
	5470	{
	5471	return XXH3_128bits_withSeed(input, len, seed);
	5472	}
	5473
	5474
	5475	/* === XXH3 128-bit streaming === */
	5476
	5477	/*
	5478	* All initialization and update functions are identical to 64-bit streaming variant.
	5479	* The only difference is the finalization routine.
	5480	*/
	5481
	5482	/! @ingroup xxh3_family /
	5483	XXH_PUBLIC_API XXH_errorcode
	5484	XXH3_128bits_reset(XXH3_state_t* statePtr)
	5485	{
	5486	return XXH3_64bits_reset(statePtr);
	5487	}
	5488
	5489	/! @ingroup xxh3_family /
	5490	XXH_PUBLIC_API XXH_errorcode
	5491	XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize)
	5492	{
	5493	return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize);
	5494	}
	5495
	5496	/! @ingroup xxh3_family /
	5497	XXH_PUBLIC_API XXH_errorcode
	5498	XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed)
	5499	{
	5500	return XXH3_64bits_reset_withSeed(statePtr, seed);
	5501	}
	5502
	5503	/! @ingroup xxh3_family /
	5504	XXH_PUBLIC_API XXH_errorcode
	5505	XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr, const void* secret, size_t secretSize, XXH64_hash_t seed)
	5506	{
	5507	return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed);
	5508	}
	5509
	5510	/! @ingroup xxh3_family /
	5511	XXH_PUBLIC_API XXH_errorcode
	5512	XXH3_128bits_update(XXH3_state_t* state, const void* input, size_t len)
	5513	{
	5514	return XXH3_update(state, (const xxh_u8*)input, len,
	5515	XXH3_accumulate_512, XXH3_scrambleAcc);
	5516	}
	5517
	5518	/! @ingroup xxh3_family /
	5519	XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* state)
	5520	{
	5521	const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
	5522	if (state->totalLen > XXH3_MIDSIZE_MAX) {
	5523	XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB];
	5524	XXH3_digest_long(acc, state, secret);
	5525	XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
	5526	{ XXH128_hash_t h128;
	5527	h128.low64 = XXH3_mergeAccs(acc,
	5528	secret + XXH_SECRET_MERGEACCS_START,
	5529	(xxh_u64)state->totalLen * XXH_PRIME64_1);
	5530	h128.high64 = XXH3_mergeAccs(acc,
	5531	secret + state->secretLimit + XXH_STRIPE_LEN
	5532	- sizeof(acc) - XXH_SECRET_MERGEACCS_START,
	5533	~((xxh_u64)state->totalLen * XXH_PRIME64_2));
	5534	return h128;
	5535	}
	5536	}
	5537	/* len <= XXH3_MIDSIZE_MAX : short code */
	5538	if (state->seed)
	5539	return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
	5540	return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen),
	5541	secret, state->secretLimit + XXH_STRIPE_LEN);
	5542	}
	5543
	5544	/* 128-bit utility functions */
	5545
	5546	#include <string.h> /* memcmp, memcpy */
	5547
	5548	/* return : 1 is equal, 0 if different */
	5549	/! @ingroup xxh3_family /
	5550	XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2)
	5551	{
	5552	/* note : XXH128_hash_t is compact, it has no padding byte */
	5553	return !(memcmp(&h1, &h2, sizeof(h1)));
	5554	}
	5555
	5556	/* This prototype is compatible with stdlib's qsort().
	5557	* return : >0 if h128_1 > h128_2
	5558	* <0 if h128_1 < h128_2
	5559	* =0 if h128_1 == h128_2 */
	5560	/! @ingroup xxh3_family /
	5561	XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2)
	5562	{
	5563	XXH128_hash_t const h1 = (const XXH128_hash_t)h128_1;
	5564	XXH128_hash_t const h2 = (const XXH128_hash_t)h128_2;
	5565	int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64);
	5566	/* note : bets that, in most cases, hash values are different */
	5567	if (hcmp) return hcmp;
	5568	return (h1.low64 > h2.low64) - (h2.low64 > h1.low64);
	5569	}
	5570
	5571
	5572	/====== Canonical representation ======/
	5573	/! @ingroup xxh3_family /
	5574	XXH_PUBLIC_API void
	5575	XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash)
	5576	{
	5577	XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t));
	5578	if (XXH_CPU_LITTLE_ENDIAN) {
	5579	hash.high64 = XXH_swap64(hash.high64);
	5580	hash.low64 = XXH_swap64(hash.low64);
	5581	}
	5582	XXH_memcpy(dst, &hash.high64, sizeof(hash.high64));
	5583	XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64));
	5584	}
	5585
	5586	/! @ingroup xxh3_family /
	5587	XXH_PUBLIC_API XXH128_hash_t
	5588	XXH128_hashFromCanonical(const XXH128_canonical_t* src)
	5589	{
	5590	XXH128_hash_t h;
	5591	h.high64 = XXH_readBE64(src);
	5592	h.low64 = XXH_readBE64(src->digest + 8);
	5593	return h;
	5594	}
	5595
	5596
	5597
	5598	/* ==========================================
	5599	* Secret generators
	5600	* ==========================================
	5601	*/
	5602	#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x))
	5603
	5604	XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128)
	5605	{
	5606	XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 );
	5607	XXH_writeLE64( (char)dst+8, XXH_readLE64((char)dst+8) ^ h128.high64 );
	5608	}
	5609
	5610	/! @ingroup xxh3_family /
	5611	XXH_PUBLIC_API XXH_errorcode
	5612	XXH3_generateSecret(void* secretBuffer, size_t secretSize, const void* customSeed, size_t customSeedSize)
	5613	{
	5614	#if (XXH_DEBUGLEVEL >= 1)
	5615	XXH_ASSERT(secretBuffer != NULL);
	5616	XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
	5617	#else
	5618	/* production mode, assert() are disabled */
	5619	if (secretBuffer == NULL) return XXH_ERROR;
	5620	if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
	5621	#endif
	5622
	5623	if (customSeedSize == 0) {
	5624	customSeed = XXH3_kSecret;
	5625	customSeedSize = XXH_SECRET_DEFAULT_SIZE;
	5626	}
	5627	#if (XXH_DEBUGLEVEL >= 1)
	5628	XXH_ASSERT(customSeed != NULL);
	5629	#else
	5630	if (customSeed == NULL) return XXH_ERROR;
	5631	#endif
	5632
	5633	/* Fill secretBuffer with a copy of customSeed - repeat as needed */
	5634	{ size_t pos = 0;
	5635	while (pos < secretSize) {
	5636	size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize);
	5637	memcpy((char*)secretBuffer + pos, customSeed, toCopy);
	5638	pos += toCopy;
	5639	} }
	5640
	5641	{ size_t const nbSeg16 = secretSize / 16;
	5642	size_t n;
	5643	XXH128_canonical_t scrambler;
	5644	XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0));
	5645	for (n=0; n<nbSeg16; n++) {
	5646	XXH128_hash_t const h128 = XXH128(&scrambler, sizeof(scrambler), n);
	5647	XXH3_combine16((char)secretBuffer + n16, h128);
	5648	}
	5649	/* last segment */
	5650	XXH3_combine16((char*)secretBuffer + secretSize - 16, XXH128_hashFromCanonical(&scrambler));
	5651	}
	5652	return XXH_OK;
	5653	}
	5654
	5655	/! @ingroup xxh3_family /
	5656	XXH_PUBLIC_API void
	5657	XXH3_generateSecret_fromSeed(void* secretBuffer, XXH64_hash_t seed)
	5658	{
	5659	XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
	5660	XXH3_initCustomSecret(secret, seed);
	5661	XXH_ASSERT(secretBuffer != NULL);
	5662	memcpy(secretBuffer, secret, XXH_SECRET_DEFAULT_SIZE);
	5663	}
	5664
	5665
	5666
	5667	/* Pop our optimization override from above */
	5668	#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \
	5669	&& defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
	5670	&& defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */
	5671	# pragma GCC pop_options
	5672	#endif
	5673
	5674	#endif /* XXH_NO_LONG_LONG */
	5675
	5676	#endif /* XXH_NO_XXH3 */
	5677
	5678	/*!
	5679	* @}
	5680	*/
	5681	#endif /* XXH_IMPLEMENTATION */
	5682
	5683
	5684	#if defined (__cplusplus)
	5685	}
	5686	#endif

stage1/zstd/lib/common/zstd_common.c created+83

...	...	@@ -0,0 +1,83 @@
	1	/*
	2	* Copyright (c) Yann Collet, Facebook, Inc.
	3	* All rights reserved.
	4	*
	5	* This source code is licensed under both the BSD-style license (found in the
	6	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	7	* in the COPYING file in the root directory of this source tree).
	8	* You may select, at your option, one of the above-listed licenses.
	9	*/
	10
	11
	12
	13	/-************************************
	14	* Dependencies
	15	***************************************/
	16	#define ZSTD_DEPS_NEED_MALLOC
	17	#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */
	18	#include "error_private.h"
	19	#include "zstd_internal.h"
	20
	21
	22	/-***************************************
	23	* Version
	24	******************************************/
	25	unsigned ZSTD_versionNumber(void) { return ZSTD_VERSION_NUMBER; }
	26
	27	const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; }
	28
	29
	30	/-***************************************
	31	* ZSTD Error Management
	32	******************************************/
	33	#undef ZSTD_isError /* defined within zstd_internal.h */
	34	/*! ZSTD_isError() :
	35	* tells if a return value is an error code
	36	* symbol is required for external callers */
	37	unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
	38
	39	/*! ZSTD_getErrorName() :
	40	* provides error code string from function result (useful for debugging) */
	41	const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); }
	42
	43	/*! ZSTD_getError() :
	44	* convert a `size_t` function result into a proper ZSTD_errorCode enum */
	45	ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
	46
	47	/*! ZSTD_getErrorString() :
	48	* provides error code string from enum */
	49	const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); }
	50
	51
	52
	53	/=*************************************************************
	54	* Custom allocator
	55	****************************************************************/
	56	void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem)
	57	{
	58	if (customMem.customAlloc)
	59	return customMem.customAlloc(customMem.opaque, size);
	60	return ZSTD_malloc(size);
	61	}
	62
	63	void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem)
	64	{
	65	if (customMem.customAlloc) {
	66	/* calloc implemented as malloc+memset;
	67	* not as efficient as calloc, but next best guess for custom malloc */
	68	void* const ptr = customMem.customAlloc(customMem.opaque, size);
	69	ZSTD_memset(ptr, 0, size);
	70	return ptr;
	71	}
	72	return ZSTD_calloc(1, size);
	73	}
	74
	75	void ZSTD_customFree(void* ptr, ZSTD_customMem customMem)
	76	{
	77	if (ptr!=NULL) {
	78	if (customMem.customFree)
	79	customMem.customFree(customMem.opaque, ptr);
	80	else
	81	ZSTD_free(ptr);
	82	}
	83	}

stage1/zstd/lib/common/zstd_deps.h created+111

...	...	@@ -0,0 +1,111 @@
	1	/*
	2	* Copyright (c) Facebook, Inc.
	3	* All rights reserved.
	4	*
	5	* This source code is licensed under both the BSD-style license (found in the
	6	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	7	* in the COPYING file in the root directory of this source tree).
	8	* You may select, at your option, one of the above-listed licenses.
	9	*/
	10
	11	/* This file provides common libc dependencies that zstd requires.
	12	* The purpose is to allow replacing this file with a custom implementation
	13	* to compile zstd without libc support.
	14	*/
	15
	16	/* Need:
	17	* NULL
	18	* INT_MAX
	19	* UINT_MAX
	20	* ZSTD_memcpy()
	21	* ZSTD_memset()
	22	* ZSTD_memmove()
	23	*/
	24	#ifndef ZSTD_DEPS_COMMON
	25	#define ZSTD_DEPS_COMMON
	26
	27	#include <limits.h>
	28	#include <stddef.h>
	29	#include <string.h>
	30
	31	#if defined(__GNUC__) && __GNUC__ >= 4
	32	# define ZSTD_memcpy(d,s,l) __builtin_memcpy((d),(s),(l))
	33	# define ZSTD_memmove(d,s,l) __builtin_memmove((d),(s),(l))
	34	# define ZSTD_memset(p,v,l) __builtin_memset((p),(v),(l))
	35	#else
	36	# define ZSTD_memcpy(d,s,l) memcpy((d),(s),(l))
	37	# define ZSTD_memmove(d,s,l) memmove((d),(s),(l))
	38	# define ZSTD_memset(p,v,l) memset((p),(v),(l))
	39	#endif
	40
	41	#endif /* ZSTD_DEPS_COMMON */
	42
	43	/* Need:
	44	* ZSTD_malloc()
	45	* ZSTD_free()
	46	* ZSTD_calloc()
	47	*/
	48	#ifdef ZSTD_DEPS_NEED_MALLOC
	49	#ifndef ZSTD_DEPS_MALLOC
	50	#define ZSTD_DEPS_MALLOC
	51
	52	#include <stdlib.h>
	53
	54	#define ZSTD_malloc(s) malloc(s)
	55	#define ZSTD_calloc(n,s) calloc((n), (s))
	56	#define ZSTD_free(p) free((p))
	57
	58	#endif /* ZSTD_DEPS_MALLOC */
	59	#endif /* ZSTD_DEPS_NEED_MALLOC */
	60
	61	/*
	62	* Provides 64-bit math support.
	63	* Need:
	64	* U64 ZSTD_div64(U64 dividend, U32 divisor)
	65	*/
	66	#ifdef ZSTD_DEPS_NEED_MATH64
	67	#ifndef ZSTD_DEPS_MATH64
	68	#define ZSTD_DEPS_MATH64
	69
	70	#define ZSTD_div64(dividend, divisor) ((dividend) / (divisor))
	71
	72	#endif /* ZSTD_DEPS_MATH64 */
	73	#endif /* ZSTD_DEPS_NEED_MATH64 */
	74
	75	/* Need:
	76	* assert()
	77	*/
	78	#ifdef ZSTD_DEPS_NEED_ASSERT
	79	#ifndef ZSTD_DEPS_ASSERT
	80	#define ZSTD_DEPS_ASSERT
	81
	82	#include <assert.h>
	83
	84	#endif /* ZSTD_DEPS_ASSERT */
	85	#endif /* ZSTD_DEPS_NEED_ASSERT */
	86
	87	/* Need:
	88	* ZSTD_DEBUG_PRINT()
	89	*/
	90	#ifdef ZSTD_DEPS_NEED_IO
	91	#ifndef ZSTD_DEPS_IO
	92	#define ZSTD_DEPS_IO
	93
	94	#include <stdio.h>
	95	#define ZSTD_DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__)
	96
	97	#endif /* ZSTD_DEPS_IO */
	98	#endif /* ZSTD_DEPS_NEED_IO */
	99
	100	/* Only requested when <stdint.h> is known to be present.
	101	* Need:
	102	* intptr_t
	103	*/
	104	#ifdef ZSTD_DEPS_NEED_STDINT
	105	#ifndef ZSTD_DEPS_STDINT
	106	#define ZSTD_DEPS_STDINT
	107
	108	#include <stdint.h>
	109
	110	#endif /* ZSTD_DEPS_STDINT */
	111	#endif /* ZSTD_DEPS_NEED_STDINT */

stage1/zstd/lib/common/zstd_internal.h created+493

...	...	@@ -0,0 +1,493 @@
	1	/*
	2	* Copyright (c) Yann Collet, Facebook, Inc.
	3	* All rights reserved.
	4	*
	5	* This source code is licensed under both the BSD-style license (found in the
	6	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	7	* in the COPYING file in the root directory of this source tree).
	8	* You may select, at your option, one of the above-listed licenses.
	9	*/
	10
	11	#ifndef ZSTD_CCOMMON_H_MODULE
	12	#define ZSTD_CCOMMON_H_MODULE
	13
	14	/* this module contains definitions which must be identical
	15	* across compression, decompression and dictBuilder.
	16	* It also contains a few functions useful to at least 2 of them
	17	* and which benefit from being inlined */
	18
	19	/-************************************
	20	* Dependencies
	21	***************************************/
	22	#include "compiler.h"
	23	#include "cpu.h"
	24	#include "mem.h"
	25	#include "debug.h" /* assert, DEBUGLOG, RAWLOG, g_debuglevel */
	26	#include "error_private.h"
	27	#define ZSTD_STATIC_LINKING_ONLY
	28	#include "../zstd.h"
	29	#define FSE_STATIC_LINKING_ONLY
	30	#include "fse.h"
	31	#define HUF_STATIC_LINKING_ONLY
	32	#include "huf.h"
	33	#ifndef XXH_STATIC_LINKING_ONLY
	34	# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
	35	#endif
	36	#include "xxhash.h" /* XXH_reset, update, digest */
	37	#ifndef ZSTD_NO_TRACE
	38	# include "zstd_trace.h"
	39	#else
	40	# define ZSTD_TRACE 0
	41	#endif
	42
	43	#if defined (__cplusplus)
	44	extern "C" {
	45	#endif
	46
	47	/* ---- static assert (debug) --- */
	48	#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c)
	49	#define ZSTD_isError ERR_isError /* for inlining */
	50	#define FSE_isError ERR_isError
	51	#define HUF_isError ERR_isError
	52
	53
	54	/-************************************
	55	* shared macros
	56	***************************************/
	57	#undef MIN
	58	#undef MAX
	59	#define MIN(a,b) ((a)<(b) ? (a) : (b))
	60	#define MAX(a,b) ((a)>(b) ? (a) : (b))
	61	#define BOUNDED(min,val,max) (MAX(min,MIN(val,max)))
	62
	63
	64	/-************************************
	65	* Common constants
	66	***************************************/
	67	#define ZSTD_OPT_NUM (1<<12)
	68
	69	#define ZSTD_REP_NUM 3 /* number of repcodes */
	70	static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
	71
	72	#define KB *(1 <<10)
	73	#define MB *(1 <<20)
	74	#define GB *(1U<<30)
	75
	76	#define BIT7 128
	77	#define BIT6 64
	78	#define BIT5 32
	79	#define BIT4 16
	80	#define BIT1 2
	81	#define BIT0 1
	82
	83	#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
	84	static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
	85	static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
	86
	87	#define ZSTD_FRAMEIDSIZE 4 /* magic number size */
	88
	89	#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
	90	static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
	91	typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
	92
	93	#define ZSTD_FRAMECHECKSUMSIZE 4
	94
	95	#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
	96	#define MIN_CBLOCK_SIZE (1 /litCSize/ + 1 /* RLE or RAW / + MIN_SEQUENCES_SIZE / nbSeq==0 /) / for a non-null block */
	97
	98	#define HufLog 12
	99	typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e;
	100
	101	#define LONGNBSEQ 0x7F00
	102
	103	#define MINMATCH 3
	104
	105	#define Litbits 8
	106	#define MaxLit ((1<<Litbits) - 1)
	107	#define MaxML 52
	108	#define MaxLL 35
	109	#define DefaultMaxOff 28
	110	#define MaxOff 31
	111	#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */
	112	#define MLFSELog 9
	113	#define LLFSELog 9
	114	#define OffFSELog 8
	115	#define MaxFSELog MAX(MAX(MLFSELog, LLFSELog), OffFSELog)
	116
	117	#define ZSTD_MAX_HUF_HEADER_SIZE 128 /* header + <= 127 byte tree description */
	118	/* Each table cannot take more than #symbols * FSELog bits */
	119	#define ZSTD_MAX_FSE_HEADERS_SIZE (((MaxML + 1) * MLFSELog + (MaxLL + 1) * LLFSELog + (MaxOff + 1) * OffFSELog + 7) / 8)
	120
	121	static UNUSED_ATTR const U8 LL_bits[MaxLL+1] = {
	122	0, 0, 0, 0, 0, 0, 0, 0,
	123	0, 0, 0, 0, 0, 0, 0, 0,
	124	1, 1, 1, 1, 2, 2, 3, 3,
	125	4, 6, 7, 8, 9,10,11,12,
	126	13,14,15,16
	127	};
	128	static UNUSED_ATTR const S16 LL_defaultNorm[MaxLL+1] = {
	129	4, 3, 2, 2, 2, 2, 2, 2,
	130	2, 2, 2, 2, 2, 1, 1, 1,
	131	2, 2, 2, 2, 2, 2, 2, 2,
	132	2, 3, 2, 1, 1, 1, 1, 1,
	133	-1,-1,-1,-1
	134	};
	135	#define LL_DEFAULTNORMLOG 6 /* for static allocation */
	136	static UNUSED_ATTR const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG;
	137
	138	static UNUSED_ATTR const U8 ML_bits[MaxML+1] = {
	139	0, 0, 0, 0, 0, 0, 0, 0,
	140	0, 0, 0, 0, 0, 0, 0, 0,
	141	0, 0, 0, 0, 0, 0, 0, 0,
	142	0, 0, 0, 0, 0, 0, 0, 0,
	143	1, 1, 1, 1, 2, 2, 3, 3,
	144	4, 4, 5, 7, 8, 9,10,11,
	145	12,13,14,15,16
	146	};
	147	static UNUSED_ATTR const S16 ML_defaultNorm[MaxML+1] = {
	148	1, 4, 3, 2, 2, 2, 2, 2,
	149	2, 1, 1, 1, 1, 1, 1, 1,
	150	1, 1, 1, 1, 1, 1, 1, 1,
	151	1, 1, 1, 1, 1, 1, 1, 1,
	152	1, 1, 1, 1, 1, 1, 1, 1,
	153	1, 1, 1, 1, 1, 1,-1,-1,
	154	-1,-1,-1,-1,-1
	155	};
	156	#define ML_DEFAULTNORMLOG 6 /* for static allocation */
	157	static UNUSED_ATTR const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG;
	158
	159	static UNUSED_ATTR const S16 OF_defaultNorm[DefaultMaxOff+1] = {
	160	1, 1, 1, 1, 1, 1, 2, 2,
	161	2, 1, 1, 1, 1, 1, 1, 1,
	162	1, 1, 1, 1, 1, 1, 1, 1,
	163	-1,-1,-1,-1,-1
	164	};
	165	#define OF_DEFAULTNORMLOG 5 /* for static allocation */
	166	static UNUSED_ATTR const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;
	167
	168
	169	/-******************************************
	170	* Shared functions to include for inlining
	171	*********************************************/
	172	static void ZSTD_copy8(void* dst, const void* src) {
	173	#if defined(ZSTD_ARCH_ARM_NEON)
	174	vst1_u8((uint8_t)dst, vld1_u8((const uint8_t)src));
	175	#else
	176	ZSTD_memcpy(dst, src, 8);
	177	#endif
	178	}
	179	#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
	180
	181	/* Need to use memmove here since the literal buffer can now be located within
	182	the dst buffer. In circumstances where the op "catches up" to where the
	183	literal buffer is, there can be partial overlaps in this call on the final
	184	copy if the literal is being shifted by less than 16 bytes. */
	185	static void ZSTD_copy16(void* dst, const void* src) {
	186	#if defined(ZSTD_ARCH_ARM_NEON)
	187	vst1q_u8((uint8_t)dst, vld1q_u8((const uint8_t)src));
	188	#elif defined(ZSTD_ARCH_X86_SSE2)
	189	_mm_storeu_si128((__m128i)dst, _mm_loadu_si128((const __m128i)src));
	190	#elif defined(__clang__)
	191	ZSTD_memmove(dst, src, 16);
	192	#else
	193	/* ZSTD_memmove is not inlined properly by gcc */
	194	BYTE copy16_buf[16];
	195	ZSTD_memcpy(copy16_buf, src, 16);
	196	ZSTD_memcpy(dst, copy16_buf, 16);
	197	#endif
	198	}
	199	#define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; }
	200
	201	#define WILDCOPY_OVERLENGTH 32
	202	#define WILDCOPY_VECLEN 16
	203
	204	typedef enum {
	205	ZSTD_no_overlap,
	206	ZSTD_overlap_src_before_dst
	207	/* ZSTD_overlap_dst_before_src, */
	208	} ZSTD_overlap_e;
	209
	210	/*! ZSTD_wildcopy() :
	211	* Custom version of ZSTD_memcpy(), can over read/write up to WILDCOPY_OVERLENGTH bytes (if length==0)
	212	* @param ovtype controls the overlap detection
	213	* - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart.
	214	* - ZSTD_overlap_src_before_dst: The src and dst may overlap, but they MUST be at least 8 bytes apart.
	215	* The src buffer must be before the dst buffer.
	216	*/
	217	MEM_STATIC FORCE_INLINE_ATTR
	218	void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e const ovtype)
	219	{
	220	ptrdiff_t diff = (BYTE)dst - (const BYTE)src;
	221	const BYTE* ip = (const BYTE*)src;
	222	BYTE* op = (BYTE*)dst;
	223	BYTE* const oend = op + length;
	224
	225	if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) {
	226	/* Handle short offset copies. */
	227	do {
	228	COPY8(op, ip)
	229	} while (op < oend);
	230	} else {
	231	assert(diff >= WILDCOPY_VECLEN \|\| diff <= -WILDCOPY_VECLEN);
	232	/* Separate out the first COPY16() call because the copy length is
	233	* almost certain to be short, so the branches have different
	234	* probabilities. Since it is almost certain to be short, only do
	235	* one COPY16() in the first call. Then, do two calls per loop since
	236	* at that point it is more likely to have a high trip count.
	237	*/
	238	#ifdef __aarch64__
	239	do {
	240	COPY16(op, ip);
	241	}
	242	while (op < oend);
	243	#else
	244	ZSTD_copy16(op, ip);
	245	if (16 >= length) return;
	246	op += 16;
	247	ip += 16;
	248	do {
	249	COPY16(op, ip);
	250	COPY16(op, ip);
	251	}
	252	while (op < oend);
	253	#endif
	254	}
	255	}
	256
	257	MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
	258	{
	259	size_t const length = MIN(dstCapacity, srcSize);
	260	if (length > 0) {
	261	ZSTD_memcpy(dst, src, length);
	262	}
	263	return length;
	264	}
	265
	266	/* define "workspace is too large" as this number of times larger than needed */
	267	#define ZSTD_WORKSPACETOOLARGE_FACTOR 3
	268
	269	/* when workspace is continuously too large
	270	* during at least this number of times,
	271	* context's memory usage is considered wasteful,
	272	* because it's sized to handle a worst case scenario which rarely happens.
	273	* In which case, resize it down to free some memory */
	274	#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128
	275
	276	/* Controls whether the input/output buffer is buffered or stable. */
	277	typedef enum {
	278	ZSTD_bm_buffered = 0, /* Buffer the input/output */
	279	ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */
	280	} ZSTD_bufferMode_e;
	281
	282
	283	/-******************************************
	284	* Private declarations
	285	*********************************************/
	286	typedef struct seqDef_s {
	287	U32 offBase; /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */
	288	U16 litLength;
	289	U16 mlBase; /* mlBase == matchLength - MINMATCH */
	290	} seqDef;
	291
	292	/* Controls whether seqStore has a single "long" litLength or matchLength. See seqStore_t. */
	293	typedef enum {
	294	ZSTD_llt_none = 0, /* no longLengthType */
	295	ZSTD_llt_literalLength = 1, /* represents a long literal */
	296	ZSTD_llt_matchLength = 2 /* represents a long match */
	297	} ZSTD_longLengthType_e;
	298
	299	typedef struct {
	300	seqDef* sequencesStart;
	301	seqDef* sequences; /* ptr to end of sequences */
	302	BYTE* litStart;
	303	BYTE* lit; /* ptr to end of literals */
	304	BYTE* llCode;
	305	BYTE* mlCode;
	306	BYTE* ofCode;
	307	size_t maxNbSeq;
	308	size_t maxNbLit;
	309
	310	/* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength
	311	* in the seqStore that has a value larger than U16 (if it exists). To do so, we increment
	312	* the existing value of the litLength or matchLength by 0x10000.
	313	*/
	314	ZSTD_longLengthType_e longLengthType;
	315	U32 longLengthPos; /* Index of the sequence to apply long length modification to */
	316	} seqStore_t;
	317
	318	typedef struct {
	319	U32 litLength;
	320	U32 matchLength;
	321	} ZSTD_sequenceLength;
	322
	323	/**
	324	* Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences
	325	* indicated by longLengthPos and longLengthType, and adds MINMATCH back to matchLength.
	326	*/
	327	MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq)
	328	{
	329	ZSTD_sequenceLength seqLen;
	330	seqLen.litLength = seq->litLength;
	331	seqLen.matchLength = seq->mlBase + MINMATCH;
	332	if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) {
	333	if (seqStore->longLengthType == ZSTD_llt_literalLength) {
	334	seqLen.litLength += 0xFFFF;
	335	}
	336	if (seqStore->longLengthType == ZSTD_llt_matchLength) {
	337	seqLen.matchLength += 0xFFFF;
	338	}
	339	}
	340	return seqLen;
	341	}
	342
	343	/**
	344	* Contains the compressed frame size and an upper-bound for the decompressed frame size.
	345	* Note: before using `compressedSize`, check for errors using ZSTD_isError().
	346	* similarly, before using `decompressedBound`, check for errors using:
	347	* `decompressedBound != ZSTD_CONTENTSIZE_ERROR`
	348	*/
	349	typedef struct {
	350	size_t compressedSize;
	351	unsigned long long decompressedBound;
	352	} ZSTD_frameSizeInfo; /* decompress & legacy */
	353
	354	const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */
	355	void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
	356
	357	/* custom memory allocation functions */
	358	void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem);
	359	void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem);
	360	void ZSTD_customFree(void* ptr, ZSTD_customMem customMem);
	361
	362
	363	MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */
	364	{
	365	assert(val != 0);
	366	{
	367	# if defined(_MSC_VER) /* Visual */
	368	# if STATIC_BMI2 == 1
	369	return _lzcnt_u32(val)^31;
	370	# else
	371	if (val != 0) {
	372	unsigned long r;
	373	_BitScanReverse(&r, val);
	374	return (unsigned)r;
	375	} else {
	376	/* Should not reach this code path */
	377	__assume(0);
	378	}
	379	# endif
	380	# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
	381	return __builtin_clz (val) ^ 31;
	382	# elif defined(__ICCARM__) /* IAR Intrinsic */
	383	return 31 - __CLZ(val);
	384	# else /* Software version */
	385	static const U32 DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
	386	U32 v = val;
	387	v \|= v >> 1;
	388	v \|= v >> 2;
	389	v \|= v >> 4;
	390	v \|= v >> 8;
	391	v \|= v >> 16;
	392	return DeBruijnClz[(v * 0x07C4ACDDU) >> 27];
	393	# endif
	394	}
	395	}
	396
	397	/**
	398	* Counts the number of trailing zeros of a `size_t`.
	399	* Most compilers should support CTZ as a builtin. A backup
	400	* implementation is provided if the builtin isn't supported, but
	401	* it may not be terribly efficient.
	402	*/
	403	MEM_STATIC unsigned ZSTD_countTrailingZeros(size_t val)
	404	{
	405	if (MEM_64bits()) {
	406	# if defined(_MSC_VER) && defined(_WIN64)
	407	# if STATIC_BMI2
	408	return _tzcnt_u64(val);
	409	# else
	410	if (val != 0) {
	411	unsigned long r;
	412	_BitScanForward64(&r, (U64)val);
	413	return (unsigned)r;
	414	} else {
	415	/* Should not reach this code path */
	416	__assume(0);
	417	}
	418	# endif
	419	# elif defined(__GNUC__) && (__GNUC__ >= 4)
	420	return __builtin_ctzll((U64)val);
	421	# else
	422	static const int DeBruijnBytePos[64] = { 0, 1, 2, 7, 3, 13, 8, 19,
	423	4, 25, 14, 28, 9, 34, 20, 56,
	424	5, 17, 26, 54, 15, 41, 29, 43,
	425	10, 31, 38, 35, 21, 45, 49, 57,
	426	63, 6, 12, 18, 24, 27, 33, 55,
	427	16, 53, 40, 42, 30, 37, 44, 48,
	428	62, 11, 23, 32, 52, 39, 36, 47,
	429	61, 22, 51, 46, 60, 50, 59, 58 };
	430	return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
	431	# endif
	432	} else { /* 32 bits */
	433	# if defined(_MSC_VER)
	434	if (val != 0) {
	435	unsigned long r;
	436	_BitScanForward(&r, (U32)val);
	437	return (unsigned)r;
	438	} else {
	439	/* Should not reach this code path */
	440	__assume(0);
	441	}
	442	# elif defined(__GNUC__) && (__GNUC__ >= 3)
	443	return __builtin_ctz((U32)val);
	444	# else
	445	static const int DeBruijnBytePos[32] = { 0, 1, 28, 2, 29, 14, 24, 3,
	446	30, 22, 20, 15, 25, 17, 4, 8,
	447	31, 27, 13, 23, 21, 19, 16, 7,
	448	26, 12, 18, 6, 11, 5, 10, 9 };
	449	return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
	450	# endif
	451	}
	452	}
	453
	454
	455	/* ZSTD_invalidateRepCodes() :
	456	* ensures next compression will not use repcodes from previous block.
	457	* Note : only works with regular variant;
	458	* do not use with extDict variant ! */
	459	void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */
	460
	461
	462	typedef struct {
	463	blockType_e blockType;
	464	U32 lastBlock;
	465	U32 origSize;
	466	} blockProperties_t; /* declared here for decompress and fullbench */
	467
	468	/*! ZSTD_getcBlockSize() :
	469	* Provides the size of compressed block from block header `src` */
	470	/* Used by: decompress, fullbench (does not get its definition from here) */
	471	size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
	472	blockProperties_t* bpPtr);
	473
	474	/*! ZSTD_decodeSeqHeaders() :
	475	* decode sequence header from src */
	476	/* Used by: decompress, fullbench (does not get its definition from here) */
	477	size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
	478	const void* src, size_t srcSize);
	479
	480	/**
	481	* @returns true iff the CPU supports dynamic BMI2 dispatch.
	482	*/
	483	MEM_STATIC int ZSTD_cpuSupportsBmi2(void)
	484	{
	485	ZSTD_cpuid_t cpuid = ZSTD_cpuid();
	486	return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid);
	487	}
	488
	489	#if defined (__cplusplus)
	490	}
	491	#endif
	492
	493	#endif /* ZSTD_CCOMMON_H_MODULE */

stage1/zstd/lib/common/zstd_trace.h created+163

...	...	@@ -0,0 +1,163 @@
	1	/*
	2	* Copyright (c) Facebook, Inc.
	3	* All rights reserved.
	4	*
	5	* This source code is licensed under both the BSD-style license (found in the
	6	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	7	* in the COPYING file in the root directory of this source tree).
	8	* You may select, at your option, one of the above-listed licenses.
	9	*/
	10
	11	#ifndef ZSTD_TRACE_H
	12	#define ZSTD_TRACE_H
	13
	14	#if defined (__cplusplus)
	15	extern "C" {
	16	#endif
	17
	18	#include <stddef.h>
	19
	20	/* weak symbol support
	21	* For now, enable conservatively:
	22	* - Only GNUC
	23	* - Only ELF
	24	* - Only x86-64 and i386
	25	* Also, explicitly disable on platforms known not to work so they aren't
	26	* forgotten in the future.
	27	*/
	28	#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && \
	29	defined(__GNUC__) && defined(__ELF__) && \
	30	(defined(__x86_64__) \|\| defined(_M_X64) \|\| defined(__i386__) \|\| defined(_M_IX86)) && \
	31	!defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \
	32	!defined(__CYGWIN__) && !defined(_AIX)
	33	# define ZSTD_HAVE_WEAK_SYMBOLS 1
	34	#else
	35	# define ZSTD_HAVE_WEAK_SYMBOLS 0
	36	#endif
	37	#if ZSTD_HAVE_WEAK_SYMBOLS
	38	# define ZSTD_WEAK_ATTR __attribute__((__weak__))
	39	#else
	40	# define ZSTD_WEAK_ATTR
	41	#endif
	42
	43	/* Only enable tracing when weak symbols are available. */
	44	#ifndef ZSTD_TRACE
	45	# define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS
	46	#endif
	47
	48	#if ZSTD_TRACE
	49
	50	struct ZSTD_CCtx_s;
	51	struct ZSTD_DCtx_s;
	52	struct ZSTD_CCtx_params_s;
	53
	54	typedef struct {
	55	/**
	56	* ZSTD_VERSION_NUMBER
	57	*
	58	* This is guaranteed to be the first member of ZSTD_trace.
	59	* Otherwise, this struct is not stable between versions. If
	60	* the version number does not match your expectation, you
	61	* should not interpret the rest of the struct.
	62	*/
	63	unsigned version;
	64	/**
	65	* Non-zero if streaming (de)compression is used.
	66	*/
	67	unsigned streaming;
	68	/**
	69	* The dictionary ID.
	70	*/
	71	unsigned dictionaryID;
	72	/**
	73	* Is the dictionary cold?
	74	* Only set on decompression.
	75	*/
	76	unsigned dictionaryIsCold;
	77	/**
	78	* The dictionary size or zero if no dictionary.
	79	*/
	80	size_t dictionarySize;
	81	/**
	82	* The uncompressed size of the data.
	83	*/
	84	size_t uncompressedSize;
	85	/**
	86	* The compressed size of the data.
	87	*/
	88	size_t compressedSize;
	89	/**
	90	* The fully resolved CCtx parameters (NULL on decompression).
	91	*/
	92	struct ZSTD_CCtx_params_s const* params;
	93	/**
	94	* The ZSTD_CCtx pointer (NULL on decompression).
	95	*/
	96	struct ZSTD_CCtx_s const* cctx;
	97	/**
	98	* The ZSTD_DCtx pointer (NULL on compression).
	99	*/
	100	struct ZSTD_DCtx_s const* dctx;
	101	} ZSTD_Trace;
	102
	103	/**
	104	* A tracing context. It must be 0 when tracing is disabled.
	105	* Otherwise, any non-zero value returned by a tracing begin()
	106	* function is presented to any subsequent calls to end().
	107	*
	108	* Any non-zero value is treated as tracing is enabled and not
	109	* interpreted by the library.
	110	*
	111	* Two possible uses are:
	112	* * A timestamp for when the begin() function was called.
	113	* * A unique key identifying the (de)compression, like the
	114	* address of the [dc]ctx pointer if you need to track
	115	* more information than just a timestamp.
	116	*/
	117	typedef unsigned long long ZSTD_TraceCtx;
	118
	119	/**
	120	* Trace the beginning of a compression call.
	121	* @param cctx The dctx pointer for the compression.
	122	* It can be used as a key to map begin() to end().
	123	* @returns Non-zero if tracing is enabled. The return value is
	124	* passed to ZSTD_trace_compress_end().
	125	*/
	126	ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin(
	127	struct ZSTD_CCtx_s const* cctx);
	128
	129	/**
	130	* Trace the end of a compression call.
	131	* @param ctx The return value of ZSTD_trace_compress_begin().
	132	* @param trace The zstd tracing info.
	133	*/
	134	ZSTD_WEAK_ATTR void ZSTD_trace_compress_end(
	135	ZSTD_TraceCtx ctx,
	136	ZSTD_Trace const* trace);
	137
	138	/**
	139	* Trace the beginning of a decompression call.
	140	* @param dctx The dctx pointer for the decompression.
	141	* It can be used as a key to map begin() to end().
	142	* @returns Non-zero if tracing is enabled. The return value is
	143	* passed to ZSTD_trace_compress_end().
	144	*/
	145	ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin(
	146	struct ZSTD_DCtx_s const* dctx);
	147
	148	/**
	149	* Trace the end of a decompression call.
	150	* @param ctx The return value of ZSTD_trace_decompress_begin().
	151	* @param trace The zstd tracing info.
	152	*/
	153	ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end(
	154	ZSTD_TraceCtx ctx,
	155	ZSTD_Trace const* trace);
	156
	157	#endif /* ZSTD_TRACE */
	158
	159	#if defined (__cplusplus)
	160	}
	161	#endif
	162
	163	#endif /* ZSTD_TRACE_H */

stage1/zstd/lib/decompress/huf_decompress.c created+1889

...	...	@@ -0,0 +1,1889 @@
	1	/* ******************************************************************
	2	* huff0 huffman decoder,
	3	* part of Finite State Entropy library
	4	* Copyright (c) Yann Collet, Facebook, Inc.
	5	*
	6	* You can contact the author at :
	7	* - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
	8	*
	9	* This source code is licensed under both the BSD-style license (found in the
	10	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	11	* in the COPYING file in the root directory of this source tree).
	12	* You may select, at your option, one of the above-listed licenses.
	13	****************************************************************** */
	14
	15	/* **************************************************************
	16	* Dependencies
	17	****************************************************************/
	18	#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */
	19	#include "../common/compiler.h"
	20	#include "../common/bitstream.h" /* BIT_* */
	21	#include "../common/fse.h" /* to compress headers */
	22	#define HUF_STATIC_LINKING_ONLY
	23	#include "../common/huf.h"
	24	#include "../common/error_private.h"
	25	#include "../common/zstd_internal.h"
	26
	27	/* **************************************************************
	28	* Constants
	29	****************************************************************/
	30
	31	#define HUF_DECODER_FAST_TABLELOG 11
	32
	33	/* **************************************************************
	34	* Macros
	35	****************************************************************/
	36
	37	/* These two optional macros force the use one way or another of the two
	38	* Huffman decompression implementations. You can't force in both directions
	39	* at the same time.
	40	*/
	41	#if defined(HUF_FORCE_DECOMPRESS_X1) && \
	42	defined(HUF_FORCE_DECOMPRESS_X2)
	43	#error "Cannot force the use of the X1 and X2 decoders at the same time!"
	44	#endif
	45
	46	#if ZSTD_ENABLE_ASM_X86_64_BMI2 && DYNAMIC_BMI2
	47	# define HUF_ASM_X86_64_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE
	48	#else
	49	# define HUF_ASM_X86_64_BMI2_ATTRS
	50	#endif
	51
	52	#ifdef __cplusplus
	53	# define HUF_EXTERN_C extern "C"
	54	#else
	55	# define HUF_EXTERN_C
	56	#endif
	57	#define HUF_ASM_DECL HUF_EXTERN_C
	58
	59	#if DYNAMIC_BMI2 \|\| (ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__))
	60	# define HUF_NEED_BMI2_FUNCTION 1
	61	#else
	62	# define HUF_NEED_BMI2_FUNCTION 0
	63	#endif
	64
	65	#if !(ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__))
	66	# define HUF_NEED_DEFAULT_FUNCTION 1
	67	#else
	68	# define HUF_NEED_DEFAULT_FUNCTION 0
	69	#endif
	70
	71	/* **************************************************************
	72	* Error Management
	73	****************************************************************/
	74	#define HUF_isError ERR_isError
	75
	76
	77	/* **************************************************************
	78	* Byte alignment for workSpace management
	79	****************************************************************/
	80	#define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a) - 1)
	81	#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask))
	82
	83
	84	/* **************************************************************
	85	* BMI2 Variant Wrappers
	86	****************************************************************/
	87	#if DYNAMIC_BMI2
	88
	89	#define HUF_DGEN(fn) \
	90	\
	91	static size_t fn##_default( \
	92	void* dst, size_t dstSize, \
	93	const void* cSrc, size_t cSrcSize, \
	94	const HUF_DTable* DTable) \
	95	{ \
	96	return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
	97	} \
	98	\
	99	static BMI2_TARGET_ATTRIBUTE size_t fn##_bmi2( \
	100	void* dst, size_t dstSize, \
	101	const void* cSrc, size_t cSrcSize, \
	102	const HUF_DTable* DTable) \
	103	{ \
	104	return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
	105	} \
	106	\
	107	static size_t fn(void* dst, size_t dstSize, void const* cSrc, \
	108	size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \
	109	{ \
	110	if (bmi2) { \
	111	return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \
	112	} \
	113	return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \
	114	}
	115
	116	#else
	117
	118	#define HUF_DGEN(fn) \
	119	static size_t fn(void* dst, size_t dstSize, void const* cSrc, \
	120	size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \
	121	{ \
	122	(void)bmi2; \
	123	return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
	124	}
	125
	126	#endif
	127
	128
	129	/-**************************/
	130	/* generic DTableDesc */
	131	/-**************************/
	132	typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc;
	133
	134	static DTableDesc HUF_getDTableDesc(const HUF_DTable* table)
	135	{
	136	DTableDesc dtd;
	137	ZSTD_memcpy(&dtd, table, sizeof(dtd));
	138	return dtd;
	139	}
	140
	141	#if ZSTD_ENABLE_ASM_X86_64_BMI2
	142
	143	static size_t HUF_initDStream(BYTE const* ip) {
	144	BYTE const lastByte = ip[7];
	145	size_t const bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
	146	size_t const value = MEM_readLEST(ip) \| 1;
	147	assert(bitsConsumed <= 8);
	148	return value << bitsConsumed;
	149	}
	150	typedef struct {
	151	BYTE const* ip[4];
	152	BYTE* op[4];
	153	U64 bits[4];
	154	void const* dt;
	155	BYTE const* ilimit;
	156	BYTE* oend;
	157	BYTE const* iend[4];
	158	} HUF_DecompressAsmArgs;
	159
	160	/**
	161	* Initializes args for the asm decoding loop.
	162	* @returns 0 on success
	163	* 1 if the fallback implementation should be used.
	164	* Or an error code on failure.
	165	*/
	166	static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable)
	167	{
	168	void const* dt = DTable + 1;
	169	U32 const dtLog = HUF_getDTableDesc(DTable).tableLog;
	170
	171	const BYTE* const ilimit = (const BYTE*)src + 6 + 8;
	172
	173	BYTE* const oend = (BYTE*)dst + dstSize;
	174
	175	/* The following condition is false on x32 platform,
	176	* but HUF_asm is not compatible with this ABI */
	177	if (!(MEM_isLittleEndian() && !MEM_32bits())) return 1;
	178
	179	/* strict minimum : jump table + 1 byte per stream */
	180	if (srcSize < 10)
	181	return ERROR(corruption_detected);
	182
	183	/* Must have at least 8 bytes per stream because we don't handle initializing smaller bit containers.
	184	* If table log is not correct at this point, fallback to the old decoder.
	185	* On small inputs we don't have enough data to trigger the fast loop, so use the old decoder.
	186	*/
	187	if (dtLog != HUF_DECODER_FAST_TABLELOG)
	188	return 1;
	189
	190	/* Read the jump table. */
	191	{
	192	const BYTE* const istart = (const BYTE*)src;
	193	size_t const length1 = MEM_readLE16(istart);
	194	size_t const length2 = MEM_readLE16(istart+2);
	195	size_t const length3 = MEM_readLE16(istart+4);
	196	size_t const length4 = srcSize - (length1 + length2 + length3 + 6);
	197	args->iend[0] = istart + 6; /* jumpTable */
	198	args->iend[1] = args->iend[0] + length1;
	199	args->iend[2] = args->iend[1] + length2;
	200	args->iend[3] = args->iend[2] + length3;
	201
	202	/* HUF_initDStream() requires this, and this small of an input
	203	* won't benefit from the ASM loop anyways.
	204	* length1 must be >= 16 so that ip[0] >= ilimit before the loop
	205	* starts.
	206	*/
	207	if (length1 < 16 \|\| length2 < 8 \|\| length3 < 8 \|\| length4 < 8)
	208	return 1;
	209	if (length4 > srcSize) return ERROR(corruption_detected); /* overflow */
	210	}
	211	/* ip[] contains the position that is currently loaded into bits[]. */
	212	args->ip[0] = args->iend[1] - sizeof(U64);
	213	args->ip[1] = args->iend[2] - sizeof(U64);
	214	args->ip[2] = args->iend[3] - sizeof(U64);
	215	args->ip[3] = (BYTE const*)src + srcSize - sizeof(U64);
	216
	217	/* op[] contains the output pointers. */
	218	args->op[0] = (BYTE*)dst;
	219	args->op[1] = args->op[0] + (dstSize+3)/4;
	220	args->op[2] = args->op[1] + (dstSize+3)/4;
	221	args->op[3] = args->op[2] + (dstSize+3)/4;
	222
	223	/* No point to call the ASM loop for tiny outputs. */
	224	if (args->op[3] >= oend)
	225	return 1;
	226
	227	/* bits[] is the bit container.
	228	* It is read from the MSB down to the LSB.
	229	* It is shifted left as it is read, and zeros are
	230	* shifted in. After the lowest valid bit a 1 is
	231	* set, so that CountTrailingZeros(bits[]) can be used
	232	* to count how many bits we've consumed.
	233	*/
	234	args->bits[0] = HUF_initDStream(args->ip[0]);
	235	args->bits[1] = HUF_initDStream(args->ip[1]);
	236	args->bits[2] = HUF_initDStream(args->ip[2]);
	237	args->bits[3] = HUF_initDStream(args->ip[3]);
	238
	239	/* If ip[] >= ilimit, it is guaranteed to be safe to
	240	* reload bits[]. It may be beyond its section, but is
	241	* guaranteed to be valid (>= istart).
	242	*/
	243	args->ilimit = ilimit;
	244
	245	args->oend = oend;
	246	args->dt = dt;
	247
	248	return 0;
	249	}
	250
	251	static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressAsmArgs const* args, int stream, BYTE* segmentEnd)
	252	{
	253	/* Validate that we haven't overwritten. */
	254	if (args->op[stream] > segmentEnd)
	255	return ERROR(corruption_detected);
	256	/* Validate that we haven't read beyond iend[].
	257	* Note that ip[] may be < iend[] because the MSB is
	258	* the next bit to read, and we may have consumed 100%
	259	* of the stream, so down to iend[i] - 8 is valid.
	260	*/
	261	if (args->ip[stream] < args->iend[stream] - 8)
	262	return ERROR(corruption_detected);
	263
	264	/* Construct the BIT_DStream_t. */
	265	bit->bitContainer = MEM_readLE64(args->ip[stream]);
	266	bit->bitsConsumed = ZSTD_countTrailingZeros((size_t)args->bits[stream]);
	267	bit->start = (const char*)args->iend[0];
	268	bit->limitPtr = bit->start + sizeof(size_t);
	269	bit->ptr = (const char*)args->ip[stream];
	270
	271	return 0;
	272	}
	273	#endif
	274
	275
	276	#ifndef HUF_FORCE_DECOMPRESS_X2
	277
	278	/-**************************/
	279	/* single-symbol decoding */
	280	/-**************************/
	281	typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1; /* single-symbol decoding */
	282
	283	/**
	284	* Packs 4 HUF_DEltX1 structs into a U64. This is used to lay down 4 entries at
	285	* a time.
	286	*/
	287	static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) {
	288	U64 D4;
	289	if (MEM_isLittleEndian()) {
	290	D4 = (symbol << 8) + nbBits;
	291	} else {
	292	D4 = symbol + (nbBits << 8);
	293	}
	294	D4 *= 0x0001000100010001ULL;
	295	return D4;
	296	}
	297
	298	/**
	299	* Increase the tableLog to targetTableLog and rescales the stats.
	300	* If tableLog > targetTableLog this is a no-op.
	301	* @returns New tableLog
	302	*/
	303	static U32 HUF_rescaleStats(BYTE* huffWeight, U32* rankVal, U32 nbSymbols, U32 tableLog, U32 targetTableLog)
	304	{
	305	if (tableLog > targetTableLog)
	306	return tableLog;
	307	if (tableLog < targetTableLog) {
	308	U32 const scale = targetTableLog - tableLog;
	309	U32 s;
	310	/* Increase the weight for all non-zero probability symbols by scale. */
	311	for (s = 0; s < nbSymbols; ++s) {
	312	huffWeight[s] += (BYTE)((huffWeight[s] == 0) ? 0 : scale);
	313	}
	314	/* Update rankVal to reflect the new weights.
	315	* All weights except 0 get moved to weight + scale.
	316	* Weights [1, scale] are empty.
	317	*/
	318	for (s = targetTableLog; s > scale; --s) {
	319	rankVal[s] = rankVal[s - scale];
	320	}
	321	for (s = scale; s > 0; --s) {
	322	rankVal[s] = 0;
	323	}
	324	}
	325	return targetTableLog;
	326	}
	327
	328	typedef struct {
	329	U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1];
	330	U32 rankStart[HUF_TABLELOG_ABSOLUTEMAX + 1];
	331	U32 statsWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
	332	BYTE symbols[HUF_SYMBOLVALUE_MAX + 1];
	333	BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
	334	} HUF_ReadDTableX1_Workspace;
	335
	336
	337	size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize)
	338	{
	339	return HUF_readDTableX1_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0);
	340	}
	341
	342	size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2)
	343	{
	344	U32 tableLog = 0;
	345	U32 nbSymbols = 0;
	346	size_t iSize;
	347	void* const dtPtr = DTable + 1;
	348	HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr;
	349	HUF_ReadDTableX1_Workspace* wksp = (HUF_ReadDTableX1_Workspace*)workSpace;
	350
	351	DEBUG_STATIC_ASSERT(HUF_DECOMPRESS_WORKSPACE_SIZE >= sizeof(*wksp));
	352	if (sizeof(*wksp) > wkspSize) return ERROR(tableLog_tooLarge);
	353
	354	DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable));
	355	/* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); / / is not necessary, even though some analyzer complain ... */
	356
	357	iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), bmi2);
	358	if (HUF_isError(iSize)) return iSize;
	359
	360
	361	/* Table header */
	362	{ DTableDesc dtd = HUF_getDTableDesc(DTable);
	363	U32 const maxTableLog = dtd.maxTableLog + 1;
	364	U32 const targetTableLog = MIN(maxTableLog, HUF_DECODER_FAST_TABLELOG);
	365	tableLog = HUF_rescaleStats(wksp->huffWeight, wksp->rankVal, nbSymbols, tableLog, targetTableLog);
	366	if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */
	367	dtd.tableType = 0;
	368	dtd.tableLog = (BYTE)tableLog;
	369	ZSTD_memcpy(DTable, &dtd, sizeof(dtd));
	370	}
	371
	372	/* Compute symbols and rankStart given rankVal:
	373	*
	374	* rankVal already contains the number of values of each weight.
	375	*
	376	* symbols contains the symbols ordered by weight. First are the rankVal[0]
	377	* weight 0 symbols, followed by the rankVal[1] weight 1 symbols, and so on.
	378	* symbols[0] is filled (but unused) to avoid a branch.
	379	*
	380	* rankStart contains the offset where each rank belongs in the DTable.
	381	* rankStart[0] is not filled because there are no entries in the table for
	382	* weight 0.
	383	*/
	384	{
	385	int n;
	386	int nextRankStart = 0;
	387	int const unroll = 4;
	388	int const nLimit = (int)nbSymbols - unroll + 1;
	389	for (n=0; n<(int)tableLog+1; n++) {
	390	U32 const curr = nextRankStart;
	391	nextRankStart += wksp->rankVal[n];
	392	wksp->rankStart[n] = curr;
	393	}
	394	for (n=0; n < nLimit; n += unroll) {
	395	int u;
	396	for (u=0; u < unroll; ++u) {
	397	size_t const w = wksp->huffWeight[n+u];
	398	wksp->symbols[wksp->rankStart[w]++] = (BYTE)(n+u);
	399	}
	400	}
	401	for (; n < (int)nbSymbols; ++n) {
	402	size_t const w = wksp->huffWeight[n];
	403	wksp->symbols[wksp->rankStart[w]++] = (BYTE)n;
	404	}
	405	}
	406
	407	/* fill DTable
	408	* We fill all entries of each weight in order.
	409	* That way length is a constant for each iteration of the outer loop.
	410	* We can switch based on the length to a different inner loop which is
	411	* optimized for that particular case.
	412	*/
	413	{
	414	U32 w;
	415	int symbol=wksp->rankVal[0];
	416	int rankStart=0;
	417	for (w=1; w<tableLog+1; ++w) {
	418	int const symbolCount = wksp->rankVal[w];
	419	int const length = (1 << w) >> 1;
	420	int uStart = rankStart;
	421	BYTE const nbBits = (BYTE)(tableLog + 1 - w);
	422	int s;
	423	int u;
	424	switch (length) {
	425	case 1:
	426	for (s=0; s<symbolCount; ++s) {
	427	HUF_DEltX1 D;
	428	D.byte = wksp->symbols[symbol + s];
	429	D.nbBits = nbBits;
	430	dt[uStart] = D;
	431	uStart += 1;
	432	}
	433	break;
	434	case 2:
	435	for (s=0; s<symbolCount; ++s) {
	436	HUF_DEltX1 D;
	437	D.byte = wksp->symbols[symbol + s];
	438	D.nbBits = nbBits;
	439	dt[uStart+0] = D;
	440	dt[uStart+1] = D;
	441	uStart += 2;
	442	}
	443	break;
	444	case 4:
	445	for (s=0; s<symbolCount; ++s) {
	446	U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits);
	447	MEM_write64(dt + uStart, D4);
	448	uStart += 4;
	449	}
	450	break;
	451	case 8:
	452	for (s=0; s<symbolCount; ++s) {
	453	U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits);
	454	MEM_write64(dt + uStart, D4);
	455	MEM_write64(dt + uStart + 4, D4);
	456	uStart += 8;
	457	}
	458	break;
	459	default:
	460	for (s=0; s<symbolCount; ++s) {
	461	U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits);
	462	for (u=0; u < length; u += 16) {
	463	MEM_write64(dt + uStart + u + 0, D4);
	464	MEM_write64(dt + uStart + u + 4, D4);
	465	MEM_write64(dt + uStart + u + 8, D4);
	466	MEM_write64(dt + uStart + u + 12, D4);
	467	}
	468	assert(u == length);
	469	uStart += length;
	470	}
	471	break;
	472	}
	473	symbol += symbolCount;
	474	rankStart += symbolCount * length;
	475	}
	476	}
	477	return iSize;
	478	}
	479
	480	FORCE_INLINE_TEMPLATE BYTE
	481	HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog)
	482	{
	483	size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */
	484	BYTE const c = dt[val].byte;
	485	BIT_skipBits(Dstream, dt[val].nbBits);
	486	return c;
	487	}
	488
	489	#define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) \
	490	*ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog)
	491
	492	#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \
	493	if (MEM_64bits() \|\| (HUF_TABLELOG_MAX<=12)) \
	494	HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr)
	495
	496	#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \
	497	if (MEM_64bits()) \
	498	HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr)
	499
	500	HINT_INLINE size_t
	501	HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog)
	502	{
	503	BYTE* const pStart = p;
	504
	505	/* up to 4 symbols at a time */
	506	if ((pEnd - p) > 3) {
	507	while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-3)) {
	508	HUF_DECODE_SYMBOLX1_2(p, bitDPtr);
	509	HUF_DECODE_SYMBOLX1_1(p, bitDPtr);
	510	HUF_DECODE_SYMBOLX1_2(p, bitDPtr);
	511	HUF_DECODE_SYMBOLX1_0(p, bitDPtr);
	512	}
	513	} else {
	514	BIT_reloadDStream(bitDPtr);
	515	}
	516
	517	/* [0-3] symbols remaining */
	518	if (MEM_32bits())
	519	while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd))
	520	HUF_DECODE_SYMBOLX1_0(p, bitDPtr);
	521
	522	/* no more data to retrieve from bitstream, no need to reload */
	523	while (p < pEnd)
	524	HUF_DECODE_SYMBOLX1_0(p, bitDPtr);
	525
	526	return pEnd-pStart;
	527	}
	528
	529	FORCE_INLINE_TEMPLATE size_t
	530	HUF_decompress1X1_usingDTable_internal_body(
	531	void* dst, size_t dstSize,
	532	const void* cSrc, size_t cSrcSize,
	533	const HUF_DTable* DTable)
	534	{
	535	BYTE* op = (BYTE*)dst;
	536	BYTE* const oend = op + dstSize;
	537	const void* dtPtr = DTable + 1;
	538	const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr;
	539	BIT_DStream_t bitD;
	540	DTableDesc const dtd = HUF_getDTableDesc(DTable);
	541	U32 const dtLog = dtd.tableLog;
	542
	543	CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) );
	544
	545	HUF_decodeStreamX1(op, &bitD, oend, dt, dtLog);
	546
	547	if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
	548
	549	return dstSize;
	550	}
	551
	552	FORCE_INLINE_TEMPLATE size_t
	553	HUF_decompress4X1_usingDTable_internal_body(
	554	void* dst, size_t dstSize,
	555	const void* cSrc, size_t cSrcSize,
	556	const HUF_DTable* DTable)
	557	{
	558	/* Check */
	559	if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
	560
	561	{ const BYTE* const istart = (const BYTE*) cSrc;
	562	BYTE* const ostart = (BYTE*) dst;
	563	BYTE* const oend = ostart + dstSize;
	564	BYTE* const olimit = oend - 3;
	565	const void* const dtPtr = DTable + 1;
	566	const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr;
	567
	568	/* Init */
	569	BIT_DStream_t bitD1;
	570	BIT_DStream_t bitD2;
	571	BIT_DStream_t bitD3;
	572	BIT_DStream_t bitD4;
	573	size_t const length1 = MEM_readLE16(istart);
	574	size_t const length2 = MEM_readLE16(istart+2);
	575	size_t const length3 = MEM_readLE16(istart+4);
	576	size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
	577	const BYTE* const istart1 = istart + 6; /* jumpTable */
	578	const BYTE* const istart2 = istart1 + length1;
	579	const BYTE* const istart3 = istart2 + length2;
	580	const BYTE* const istart4 = istart3 + length3;
	581	const size_t segmentSize = (dstSize+3) / 4;
	582	BYTE* const opStart2 = ostart + segmentSize;
	583	BYTE* const opStart3 = opStart2 + segmentSize;
	584	BYTE* const opStart4 = opStart3 + segmentSize;
	585	BYTE* op1 = ostart;
	586	BYTE* op2 = opStart2;
	587	BYTE* op3 = opStart3;
	588	BYTE* op4 = opStart4;
	589	DTableDesc const dtd = HUF_getDTableDesc(DTable);
	590	U32 const dtLog = dtd.tableLog;
	591	U32 endSignal = 1;
	592
	593	if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
	594	if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */
	595	CHECK_F( BIT_initDStream(&bitD1, istart1, length1) );
	596	CHECK_F( BIT_initDStream(&bitD2, istart2, length2) );
	597	CHECK_F( BIT_initDStream(&bitD3, istart3, length3) );
	598	CHECK_F( BIT_initDStream(&bitD4, istart4, length4) );
	599
	600	/* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */
	601	if ((size_t)(oend - op4) >= sizeof(size_t)) {
	602	for ( ; (endSignal) & (op4 < olimit) ; ) {
	603	HUF_DECODE_SYMBOLX1_2(op1, &bitD1);
	604	HUF_DECODE_SYMBOLX1_2(op2, &bitD2);
	605	HUF_DECODE_SYMBOLX1_2(op3, &bitD3);
	606	HUF_DECODE_SYMBOLX1_2(op4, &bitD4);
	607	HUF_DECODE_SYMBOLX1_1(op1, &bitD1);
	608	HUF_DECODE_SYMBOLX1_1(op2, &bitD2);
	609	HUF_DECODE_SYMBOLX1_1(op3, &bitD3);
	610	HUF_DECODE_SYMBOLX1_1(op4, &bitD4);
	611	HUF_DECODE_SYMBOLX1_2(op1, &bitD1);
	612	HUF_DECODE_SYMBOLX1_2(op2, &bitD2);
	613	HUF_DECODE_SYMBOLX1_2(op3, &bitD3);
	614	HUF_DECODE_SYMBOLX1_2(op4, &bitD4);
	615	HUF_DECODE_SYMBOLX1_0(op1, &bitD1);
	616	HUF_DECODE_SYMBOLX1_0(op2, &bitD2);
	617	HUF_DECODE_SYMBOLX1_0(op3, &bitD3);
	618	HUF_DECODE_SYMBOLX1_0(op4, &bitD4);
	619	endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished;
	620	endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished;
	621	endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished;
	622	endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished;
	623	}
	624	}
	625
	626	/* check corruption */
	627	/* note : should not be necessary : op# advance in lock step, and we control op4.
	628	* but curiously, binary generated by gcc 7.2 & 7.3 with -mbmi2 runs faster when >=1 test is present */
	629	if (op1 > opStart2) return ERROR(corruption_detected);
	630	if (op2 > opStart3) return ERROR(corruption_detected);
	631	if (op3 > opStart4) return ERROR(corruption_detected);
	632	/* note : op4 supposed already verified within main loop */
	633
	634	/* finish bitStreams one by one */
	635	HUF_decodeStreamX1(op1, &bitD1, opStart2, dt, dtLog);
	636	HUF_decodeStreamX1(op2, &bitD2, opStart3, dt, dtLog);
	637	HUF_decodeStreamX1(op3, &bitD3, opStart4, dt, dtLog);
	638	HUF_decodeStreamX1(op4, &bitD4, oend, dt, dtLog);
	639
	640	/* check */
	641	{ U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
	642	if (!endCheck) return ERROR(corruption_detected); }
	643
	644	/* decoded size */
	645	return dstSize;
	646	}
	647	}
	648
	649	#if HUF_NEED_BMI2_FUNCTION
	650	static BMI2_TARGET_ATTRIBUTE
	651	size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc,
	652	size_t cSrcSize, HUF_DTable const* DTable) {
	653	return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
	654	}
	655	#endif
	656
	657	#if HUF_NEED_DEFAULT_FUNCTION
	658	static
	659	size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc,
	660	size_t cSrcSize, HUF_DTable const* DTable) {
	661	return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
	662	}
	663	#endif
	664
	665	#if ZSTD_ENABLE_ASM_X86_64_BMI2
	666
	667	HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop(HUF_DecompressAsmArgs* args) ZSTDLIB_HIDDEN;
	668
	669	static HUF_ASM_X86_64_BMI2_ATTRS
	670	size_t
	671	HUF_decompress4X1_usingDTable_internal_bmi2_asm(
	672	void* dst, size_t dstSize,
	673	const void* cSrc, size_t cSrcSize,
	674	const HUF_DTable* DTable)
	675	{
	676	void const* dt = DTable + 1;
	677	const BYTE* const iend = (const BYTE*)cSrc + 6;
	678	BYTE* const oend = (BYTE*)dst + dstSize;
	679	HUF_DecompressAsmArgs args;
	680	{
	681	size_t const ret = HUF_DecompressAsmArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
	682	FORWARD_IF_ERROR(ret, "Failed to init asm args");
	683	if (ret != 0)
	684	return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
	685	}
	686
	687	assert(args.ip[0] >= args.ilimit);
	688	HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop(&args);
	689
	690	/* Our loop guarantees that ip[] >= ilimit and that we haven't
	691	* overwritten any op[].
	692	*/
	693	assert(args.ip[0] >= iend);
	694	assert(args.ip[1] >= iend);
	695	assert(args.ip[2] >= iend);
	696	assert(args.ip[3] >= iend);
	697	assert(args.op[3] <= oend);
	698	(void)iend;
	699
	700	/* finish bit streams one by one. */
	701	{
	702	size_t const segmentSize = (dstSize+3) / 4;
	703	BYTE* segmentEnd = (BYTE*)dst;
	704	int i;
	705	for (i = 0; i < 4; ++i) {
	706	BIT_DStream_t bit;
	707	if (segmentSize <= (size_t)(oend - segmentEnd))
	708	segmentEnd += segmentSize;
	709	else
	710	segmentEnd = oend;
	711	FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption");
	712	/* Decompress and validate that we've produced exactly the expected length. */
	713	args.op[i] += HUF_decodeStreamX1(args.op[i], &bit, segmentEnd, (HUF_DEltX1 const*)dt, HUF_DECODER_FAST_TABLELOG);
	714	if (args.op[i] != segmentEnd) return ERROR(corruption_detected);
	715	}
	716	}
	717
	718	/* decoded size */
	719	return dstSize;
	720	}
	721	#endif /* ZSTD_ENABLE_ASM_X86_64_BMI2 */
	722
	723	typedef size_t (HUF_decompress_usingDTable_t)(void dst, size_t dstSize,
	724	const void *cSrc,
	725	size_t cSrcSize,
	726	const HUF_DTable *DTable);
	727
	728	HUF_DGEN(HUF_decompress1X1_usingDTable_internal)
	729
	730	static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc,
	731	size_t cSrcSize, HUF_DTable const* DTable, int bmi2)
	732	{
	733	#if DYNAMIC_BMI2
	734	if (bmi2) {
	735	# if ZSTD_ENABLE_ASM_X86_64_BMI2
	736	return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
	737	# else
	738	return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
	739	# endif
	740	}
	741	#else
	742	(void)bmi2;
	743	#endif
	744
	745	#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
	746	return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
	747	#else
	748	return HUF_decompress4X1_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable);
	749	#endif
	750	}
	751
	752
	753	size_t HUF_decompress1X1_usingDTable(
	754	void* dst, size_t dstSize,
	755	const void* cSrc, size_t cSrcSize,
	756	const HUF_DTable* DTable)
	757	{
	758	DTableDesc dtd = HUF_getDTableDesc(DTable);
	759	if (dtd.tableType != 0) return ERROR(GENERIC);
	760	return HUF_decompress1X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
	761	}
	762
	763	size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize,
	764	const void* cSrc, size_t cSrcSize,
	765	void* workSpace, size_t wkspSize)
	766	{
	767	const BYTE* ip = (const BYTE*) cSrc;
	768
	769	size_t const hSize = HUF_readDTableX1_wksp(DCtx, cSrc, cSrcSize, workSpace, wkspSize);
	770	if (HUF_isError(hSize)) return hSize;
	771	if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
	772	ip += hSize; cSrcSize -= hSize;
	773
	774	return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0);
	775	}
	776
	777
	778	size_t HUF_decompress4X1_usingDTable(
	779	void* dst, size_t dstSize,
	780	const void* cSrc, size_t cSrcSize,
	781	const HUF_DTable* DTable)
	782	{
	783	DTableDesc dtd = HUF_getDTableDesc(DTable);
	784	if (dtd.tableType != 0) return ERROR(GENERIC);
	785	return HUF_decompress4X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
	786	}
	787
	788	static size_t HUF_decompress4X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize,
	789	const void* cSrc, size_t cSrcSize,
	790	void* workSpace, size_t wkspSize, int bmi2)
	791	{
	792	const BYTE* ip = (const BYTE*) cSrc;
	793
	794	size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
	795	if (HUF_isError(hSize)) return hSize;
	796	if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
	797	ip += hSize; cSrcSize -= hSize;
	798
	799	return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2);
	800	}
	801
	802	size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
	803	const void* cSrc, size_t cSrcSize,
	804	void* workSpace, size_t wkspSize)
	805	{
	806	return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, 0);
	807	}
	808
	809
	810	#endif /* HUF_FORCE_DECOMPRESS_X2 */
	811
	812
	813	#ifndef HUF_FORCE_DECOMPRESS_X1
	814
	815	/* *************************/
	816	/* double-symbols decoding */
	817	/* *************************/
	818
	819	typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX2; /* double-symbols decoding */
	820	typedef struct { BYTE symbol; } sortedSymbol_t;
	821	typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1];
	822	typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX];
	823
	824	/**
	825	* Constructs a HUF_DEltX2 in a U32.
	826	*/
	827	static U32 HUF_buildDEltX2U32(U32 symbol, U32 nbBits, U32 baseSeq, int level)
	828	{
	829	U32 seq;
	830	DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, sequence) == 0);
	831	DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, nbBits) == 2);
	832	DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, length) == 3);
	833	DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U32));
	834	if (MEM_isLittleEndian()) {
	835	seq = level == 1 ? symbol : (baseSeq + (symbol << 8));
	836	return seq + (nbBits << 16) + ((U32)level << 24);
	837	} else {
	838	seq = level == 1 ? (symbol << 8) : ((baseSeq << 8) + symbol);
	839	return (seq << 16) + (nbBits << 8) + (U32)level;
	840	}
	841	}
	842
	843	/**
	844	* Constructs a HUF_DEltX2.
	845	*/
	846	static HUF_DEltX2 HUF_buildDEltX2(U32 symbol, U32 nbBits, U32 baseSeq, int level)
	847	{
	848	HUF_DEltX2 DElt;
	849	U32 const val = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level);
	850	DEBUG_STATIC_ASSERT(sizeof(DElt) == sizeof(val));
	851	ZSTD_memcpy(&DElt, &val, sizeof(val));
	852	return DElt;
	853	}
	854
	855	/**
	856	* Constructs 2 HUF_DEltX2s and packs them into a U64.
	857	*/
	858	static U64 HUF_buildDEltX2U64(U32 symbol, U32 nbBits, U16 baseSeq, int level)
	859	{
	860	U32 DElt = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level);
	861	return (U64)DElt + ((U64)DElt << 32);
	862	}
	863
	864	/**
	865	* Fills the DTable rank with all the symbols from [begin, end) that are each
	866	* nbBits long.
	867	*
	868	* @param DTableRank The start of the rank in the DTable.
	869	* @param begin The first symbol to fill (inclusive).
	870	* @param end The last symbol to fill (exclusive).
	871	* @param nbBits Each symbol is nbBits long.
	872	* @param tableLog The table log.
	873	* @param baseSeq If level == 1 { 0 } else { the first level symbol }
	874	* @param level The level in the table. Must be 1 or 2.
	875	*/
	876	static void HUF_fillDTableX2ForWeight(
	877	HUF_DEltX2* DTableRank,
	878	sortedSymbol_t const* begin, sortedSymbol_t const* end,
	879	U32 nbBits, U32 tableLog,
	880	U16 baseSeq, int const level)
	881	{
	882	U32 const length = 1U << ((tableLog - nbBits) & 0x1F /* quiet static-analyzer */);
	883	const sortedSymbol_t* ptr;
	884	assert(level >= 1 && level <= 2);
	885	switch (length) {
	886	case 1:
	887	for (ptr = begin; ptr != end; ++ptr) {
	888	HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level);
	889	*DTableRank++ = DElt;
	890	}
	891	break;
	892	case 2:
	893	for (ptr = begin; ptr != end; ++ptr) {
	894	HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level);
	895	DTableRank[0] = DElt;
	896	DTableRank[1] = DElt;
	897	DTableRank += 2;
	898	}
	899	break;
	900	case 4:
	901	for (ptr = begin; ptr != end; ++ptr) {
	902	U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level);
	903	ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2));
	904	ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2));
	905	DTableRank += 4;
	906	}
	907	break;
	908	case 8:
	909	for (ptr = begin; ptr != end; ++ptr) {
	910	U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level);
	911	ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2));
	912	ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2));
	913	ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2));
	914	ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2));
	915	DTableRank += 8;
	916	}
	917	break;
	918	default:
	919	for (ptr = begin; ptr != end; ++ptr) {
	920	U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level);
	921	HUF_DEltX2* const DTableRankEnd = DTableRank + length;
	922	for (; DTableRank != DTableRankEnd; DTableRank += 8) {
	923	ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2));
	924	ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2));
	925	ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2));
	926	ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2));
	927	}
	928	}
	929	break;
	930	}
	931	}
	932
	933	/* HUF_fillDTableX2Level2() :
	934	* `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */
	935	static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 targetLog, const U32 consumedBits,
	936	const U32* rankVal, const int minWeight, const int maxWeight1,
	937	const sortedSymbol_t* sortedSymbols, U32 const* rankStart,
	938	U32 nbBitsBaseline, U16 baseSeq)
	939	{
	940	/* Fill skipped values (all positions up to rankVal[minWeight]).
	941	* These are positions only get a single symbol because the combined weight
	942	* is too large.
	943	*/
	944	if (minWeight>1) {
	945	U32 const length = 1U << ((targetLog - consumedBits) & 0x1F /* quiet static-analyzer */);
	946	U64 const DEltX2 = HUF_buildDEltX2U64(baseSeq, consumedBits, /* baseSeq / 0, / level */ 1);
	947	int const skipSize = rankVal[minWeight];
	948	assert(length > 1);
	949	assert((U32)skipSize < length);
	950	switch (length) {
	951	case 2:
	952	assert(skipSize == 1);
	953	ZSTD_memcpy(DTable, &DEltX2, sizeof(DEltX2));
	954	break;
	955	case 4:
	956	assert(skipSize <= 4);
	957	ZSTD_memcpy(DTable + 0, &DEltX2, sizeof(DEltX2));
	958	ZSTD_memcpy(DTable + 2, &DEltX2, sizeof(DEltX2));
	959	break;
	960	default:
	961	{
	962	int i;
	963	for (i = 0; i < skipSize; i += 8) {
	964	ZSTD_memcpy(DTable + i + 0, &DEltX2, sizeof(DEltX2));
	965	ZSTD_memcpy(DTable + i + 2, &DEltX2, sizeof(DEltX2));
	966	ZSTD_memcpy(DTable + i + 4, &DEltX2, sizeof(DEltX2));
	967	ZSTD_memcpy(DTable + i + 6, &DEltX2, sizeof(DEltX2));
	968	}
	969	}
	970	}
	971	}
	972
	973	/* Fill each of the second level symbols by weight. */
	974	{
	975	int w;
	976	for (w = minWeight; w < maxWeight1; ++w) {
	977	int const begin = rankStart[w];
	978	int const end = rankStart[w+1];
	979	U32 const nbBits = nbBitsBaseline - w;
	980	U32 const totalBits = nbBits + consumedBits;
	981	HUF_fillDTableX2ForWeight(
	982	DTable + rankVal[w],
	983	sortedSymbols + begin, sortedSymbols + end,
	984	totalBits, targetLog,
	985	baseSeq, /* level */ 2);
	986	}
	987	}
	988	}
	989
	990	static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog,
	991	const sortedSymbol_t* sortedList,
	992	const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight,
	993	const U32 nbBitsBaseline)
	994	{
	995	U32* const rankVal = rankValOrigin[0];
	996	const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */
	997	const U32 minBits = nbBitsBaseline - maxWeight;
	998	int w;
	999	int const wEnd = (int)maxWeight + 1;
	1000
	1001	/* Fill DTable in order of weight. */
	1002	for (w = 1; w < wEnd; ++w) {
	1003	int const begin = (int)rankStart[w];
	1004	int const end = (int)rankStart[w+1];
	1005	U32 const nbBits = nbBitsBaseline - w;
	1006
	1007	if (targetLog-nbBits >= minBits) {
	1008	/* Enough room for a second symbol. */
	1009	int start = rankVal[w];
	1010	U32 const length = 1U << ((targetLog - nbBits) & 0x1F /* quiet static-analyzer */);
	1011	int minWeight = nbBits + scaleLog;
	1012	int s;
	1013	if (minWeight < 1) minWeight = 1;
	1014	/* Fill the DTable for every symbol of weight w.
	1015	* These symbols get at least 1 second symbol.
	1016	*/
	1017	for (s = begin; s != end; ++s) {
	1018	HUF_fillDTableX2Level2(
	1019	DTable + start, targetLog, nbBits,
	1020	rankValOrigin[nbBits], minWeight, wEnd,
	1021	sortedList, rankStart,
	1022	nbBitsBaseline, sortedList[s].symbol);
	1023	start += length;
	1024	}
	1025	} else {
	1026	/* Only a single symbol. */
	1027	HUF_fillDTableX2ForWeight(
	1028	DTable + rankVal[w],
	1029	sortedList + begin, sortedList + end,
	1030	nbBits, targetLog,
	1031	/* baseSeq / 0, / level */ 1);
	1032	}
	1033	}
	1034	}
	1035
	1036	typedef struct {
	1037	rankValCol_t rankVal[HUF_TABLELOG_MAX];
	1038	U32 rankStats[HUF_TABLELOG_MAX + 1];
	1039	U32 rankStart0[HUF_TABLELOG_MAX + 3];
	1040	sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1];
	1041	BYTE weightList[HUF_SYMBOLVALUE_MAX + 1];
	1042	U32 calleeWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
	1043	} HUF_ReadDTableX2_Workspace;
	1044
	1045	size_t HUF_readDTableX2_wksp(HUF_DTable* DTable,
	1046	const void* src, size_t srcSize,
	1047	void* workSpace, size_t wkspSize)
	1048	{
	1049	return HUF_readDTableX2_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0);
	1050	}
	1051
	1052	size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable,
	1053	const void* src, size_t srcSize,
	1054	void* workSpace, size_t wkspSize, int bmi2)
	1055	{
	1056	U32 tableLog, maxW, nbSymbols;
	1057	DTableDesc dtd = HUF_getDTableDesc(DTable);
	1058	U32 maxTableLog = dtd.maxTableLog;
	1059	size_t iSize;
	1060	void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */
	1061	HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr;
	1062	U32 *rankStart;
	1063
	1064	HUF_ReadDTableX2_Workspace* const wksp = (HUF_ReadDTableX2_Workspace*)workSpace;
	1065
	1066	if (sizeof(*wksp) > wkspSize) return ERROR(GENERIC);
	1067
	1068	rankStart = wksp->rankStart0 + 1;
	1069	ZSTD_memset(wksp->rankStats, 0, sizeof(wksp->rankStats));
	1070	ZSTD_memset(wksp->rankStart0, 0, sizeof(wksp->rankStart0));
	1071
	1072	DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */
	1073	if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
	1074	/* ZSTD_memset(weightList, 0, sizeof(weightList)); / / is not necessary, even though some analyzer complain ... */
	1075
	1076	iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), bmi2);
	1077	if (HUF_isError(iSize)) return iSize;
	1078
	1079	/* check result */
	1080	if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */
	1081	if (tableLog <= HUF_DECODER_FAST_TABLELOG && maxTableLog > HUF_DECODER_FAST_TABLELOG) maxTableLog = HUF_DECODER_FAST_TABLELOG;
	1082
	1083	/* find maxWeight */
	1084	for (maxW = tableLog; wksp->rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */
	1085
	1086	/* Get start index of each weight */
	1087	{ U32 w, nextRankStart = 0;
	1088	for (w=1; w<maxW+1; w++) {
	1089	U32 curr = nextRankStart;
	1090	nextRankStart += wksp->rankStats[w];
	1091	rankStart[w] = curr;
	1092	}
	1093	rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/
	1094	rankStart[maxW+1] = nextRankStart;
	1095	}
	1096
	1097	/* sort symbols by weight */
	1098	{ U32 s;
	1099	for (s=0; s<nbSymbols; s++) {
	1100	U32 const w = wksp->weightList[s];
	1101	U32 const r = rankStart[w]++;
	1102	wksp->sortedSymbol[r].symbol = (BYTE)s;
	1103	}
	1104	rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
	1105	}
	1106
	1107	/* Build rankVal */
	1108	{ U32* const rankVal0 = wksp->rankVal[0];
	1109	{ int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */
	1110	U32 nextRankVal = 0;
	1111	U32 w;
	1112	for (w=1; w<maxW+1; w++) {
	1113	U32 curr = nextRankVal;
	1114	nextRankVal += wksp->rankStats[w] << (w+rescale);
	1115	rankVal0[w] = curr;
	1116	} }
	1117	{ U32 const minBits = tableLog+1 - maxW;
	1118	U32 consumed;
	1119	for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) {
	1120	U32* const rankValPtr = wksp->rankVal[consumed];
	1121	U32 w;
	1122	for (w = 1; w < maxW+1; w++) {
	1123	rankValPtr[w] = rankVal0[w] >> consumed;
	1124	} } } }
	1125
	1126	HUF_fillDTableX2(dt, maxTableLog,
	1127	wksp->sortedSymbol,
	1128	wksp->rankStart0, wksp->rankVal, maxW,
	1129	tableLog+1);
	1130
	1131	dtd.tableLog = (BYTE)maxTableLog;
	1132	dtd.tableType = 1;
	1133	ZSTD_memcpy(DTable, &dtd, sizeof(dtd));
	1134	return iSize;
	1135	}
	1136
	1137
	1138	FORCE_INLINE_TEMPLATE U32
	1139	HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog)
	1140	{
	1141	size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
	1142	ZSTD_memcpy(op, &dt[val].sequence, 2);
	1143	BIT_skipBits(DStream, dt[val].nbBits);
	1144	return dt[val].length;
	1145	}
	1146
	1147	FORCE_INLINE_TEMPLATE U32
	1148	HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog)
	1149	{
	1150	size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
	1151	ZSTD_memcpy(op, &dt[val].sequence, 1);
	1152	if (dt[val].length==1) {
	1153	BIT_skipBits(DStream, dt[val].nbBits);
	1154	} else {
	1155	if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) {
	1156	BIT_skipBits(DStream, dt[val].nbBits);
	1157	if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8))
	1158	/* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
	1159	DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8);
	1160	}
	1161	}
	1162	return 1;
	1163	}
	1164
	1165	#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \
	1166	ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog)
	1167
	1168	#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \
	1169	if (MEM_64bits() \|\| (HUF_TABLELOG_MAX<=12)) \
	1170	ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog)
	1171
	1172	#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \
	1173	if (MEM_64bits()) \
	1174	ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog)
	1175
	1176	HINT_INLINE size_t
	1177	HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd,
	1178	const HUF_DEltX2* const dt, const U32 dtLog)
	1179	{
	1180	BYTE* const pStart = p;
	1181
	1182	/* up to 8 symbols at a time */
	1183	if ((size_t)(pEnd - p) >= sizeof(bitDPtr->bitContainer)) {
	1184	if (dtLog <= 11 && MEM_64bits()) {
	1185	/* up to 10 symbols at a time */
	1186	while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-9)) {
	1187	HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
	1188	HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
	1189	HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
	1190	HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
	1191	HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
	1192	}
	1193	} else {
	1194	/* up to 8 symbols at a time */
	1195	while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) {
	1196	HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
	1197	HUF_DECODE_SYMBOLX2_1(p, bitDPtr);
	1198	HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
	1199	HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
	1200	}
	1201	}
	1202	} else {
	1203	BIT_reloadDStream(bitDPtr);
	1204	}
	1205
	1206	/* closer to end : up to 2 symbols at a time */
	1207	if ((size_t)(pEnd - p) >= 2) {
	1208	while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2))
	1209	HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
	1210
	1211	while (p <= pEnd-2)
	1212	HUF_DECODE_SYMBOLX2_0(p, bitDPtr); /* no need to reload : reached the end of DStream */
	1213	}
	1214
	1215	if (p < pEnd)
	1216	p += HUF_decodeLastSymbolX2(p, bitDPtr, dt, dtLog);
	1217
	1218	return p-pStart;
	1219	}
	1220
	1221	FORCE_INLINE_TEMPLATE size_t
	1222	HUF_decompress1X2_usingDTable_internal_body(
	1223	void* dst, size_t dstSize,
	1224	const void* cSrc, size_t cSrcSize,
	1225	const HUF_DTable* DTable)
	1226	{
	1227	BIT_DStream_t bitD;
	1228
	1229	/* Init */
	1230	CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) );
	1231
	1232	/* decode */
	1233	{ BYTE* const ostart = (BYTE*) dst;
	1234	BYTE* const oend = ostart + dstSize;
	1235	const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */
	1236	const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr;
	1237	DTableDesc const dtd = HUF_getDTableDesc(DTable);
	1238	HUF_decodeStreamX2(ostart, &bitD, oend, dt, dtd.tableLog);
	1239	}
	1240
	1241	/* check */
	1242	if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
	1243
	1244	/* decoded size */
	1245	return dstSize;
	1246	}
	1247	FORCE_INLINE_TEMPLATE size_t
	1248	HUF_decompress4X2_usingDTable_internal_body(
	1249	void* dst, size_t dstSize,
	1250	const void* cSrc, size_t cSrcSize,
	1251	const HUF_DTable* DTable)
	1252	{
	1253	if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
	1254
	1255	{ const BYTE* const istart = (const BYTE*) cSrc;
	1256	BYTE* const ostart = (BYTE*) dst;
	1257	BYTE* const oend = ostart + dstSize;
	1258	BYTE* const olimit = oend - (sizeof(size_t)-1);
	1259	const void* const dtPtr = DTable+1;
	1260	const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr;
	1261
	1262	/* Init */
	1263	BIT_DStream_t bitD1;
	1264	BIT_DStream_t bitD2;
	1265	BIT_DStream_t bitD3;
	1266	BIT_DStream_t bitD4;
	1267	size_t const length1 = MEM_readLE16(istart);
	1268	size_t const length2 = MEM_readLE16(istart+2);
	1269	size_t const length3 = MEM_readLE16(istart+4);
	1270	size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
	1271	const BYTE* const istart1 = istart + 6; /* jumpTable */
	1272	const BYTE* const istart2 = istart1 + length1;
	1273	const BYTE* const istart3 = istart2 + length2;
	1274	const BYTE* const istart4 = istart3 + length3;
	1275	size_t const segmentSize = (dstSize+3) / 4;
	1276	BYTE* const opStart2 = ostart + segmentSize;
	1277	BYTE* const opStart3 = opStart2 + segmentSize;
	1278	BYTE* const opStart4 = opStart3 + segmentSize;
	1279	BYTE* op1 = ostart;
	1280	BYTE* op2 = opStart2;
	1281	BYTE* op3 = opStart3;
	1282	BYTE* op4 = opStart4;
	1283	U32 endSignal = 1;
	1284	DTableDesc const dtd = HUF_getDTableDesc(DTable);
	1285	U32 const dtLog = dtd.tableLog;
	1286
	1287	if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
	1288	if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */
	1289	CHECK_F( BIT_initDStream(&bitD1, istart1, length1) );
	1290	CHECK_F( BIT_initDStream(&bitD2, istart2, length2) );
	1291	CHECK_F( BIT_initDStream(&bitD3, istart3, length3) );
	1292	CHECK_F( BIT_initDStream(&bitD4, istart4, length4) );
	1293
	1294	/* 16-32 symbols per loop (4-8 symbols per stream) */
	1295	if ((size_t)(oend - op4) >= sizeof(size_t)) {
	1296	for ( ; (endSignal) & (op4 < olimit); ) {
	1297	#if defined(__clang__) && (defined(__x86_64__) \|\| defined(__i386__))
	1298	HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
	1299	HUF_DECODE_SYMBOLX2_1(op1, &bitD1);
	1300	HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
	1301	HUF_DECODE_SYMBOLX2_0(op1, &bitD1);
	1302	HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
	1303	HUF_DECODE_SYMBOLX2_1(op2, &bitD2);
	1304	HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
	1305	HUF_DECODE_SYMBOLX2_0(op2, &bitD2);
	1306	endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished;
	1307	endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished;
	1308	HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
	1309	HUF_DECODE_SYMBOLX2_1(op3, &bitD3);
	1310	HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
	1311	HUF_DECODE_SYMBOLX2_0(op3, &bitD3);
	1312	HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
	1313	HUF_DECODE_SYMBOLX2_1(op4, &bitD4);
	1314	HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
	1315	HUF_DECODE_SYMBOLX2_0(op4, &bitD4);
	1316	endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished;
	1317	endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished;
	1318	#else
	1319	HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
	1320	HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
	1321	HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
	1322	HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
	1323	HUF_DECODE_SYMBOLX2_1(op1, &bitD1);
	1324	HUF_DECODE_SYMBOLX2_1(op2, &bitD2);
	1325	HUF_DECODE_SYMBOLX2_1(op3, &bitD3);
	1326	HUF_DECODE_SYMBOLX2_1(op4, &bitD4);
	1327	HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
	1328	HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
	1329	HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
	1330	HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
	1331	HUF_DECODE_SYMBOLX2_0(op1, &bitD1);
	1332	HUF_DECODE_SYMBOLX2_0(op2, &bitD2);
	1333	HUF_DECODE_SYMBOLX2_0(op3, &bitD3);
	1334	HUF_DECODE_SYMBOLX2_0(op4, &bitD4);
	1335	endSignal = (U32)LIKELY((U32)
	1336	(BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished)
	1337	& (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished)
	1338	& (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished)
	1339	& (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished));
	1340	#endif
	1341	}
	1342	}
	1343
	1344	/* check corruption */
	1345	if (op1 > opStart2) return ERROR(corruption_detected);
	1346	if (op2 > opStart3) return ERROR(corruption_detected);
	1347	if (op3 > opStart4) return ERROR(corruption_detected);
	1348	/* note : op4 already verified within main loop */
	1349
	1350	/* finish bitStreams one by one */
	1351	HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
	1352	HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
	1353	HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
	1354	HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog);
	1355
	1356	/* check */
	1357	{ U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
	1358	if (!endCheck) return ERROR(corruption_detected); }
	1359
	1360	/* decoded size */
	1361	return dstSize;
	1362	}
	1363	}
	1364
	1365	#if HUF_NEED_BMI2_FUNCTION
	1366	static BMI2_TARGET_ATTRIBUTE
	1367	size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc,
	1368	size_t cSrcSize, HUF_DTable const* DTable) {
	1369	return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
	1370	}
	1371	#endif
	1372
	1373	#if HUF_NEED_DEFAULT_FUNCTION
	1374	static
	1375	size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc,
	1376	size_t cSrcSize, HUF_DTable const* DTable) {
	1377	return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
	1378	}
	1379	#endif
	1380
	1381	#if ZSTD_ENABLE_ASM_X86_64_BMI2
	1382
	1383	HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop(HUF_DecompressAsmArgs* args) ZSTDLIB_HIDDEN;
	1384
	1385	static HUF_ASM_X86_64_BMI2_ATTRS size_t
	1386	HUF_decompress4X2_usingDTable_internal_bmi2_asm(
	1387	void* dst, size_t dstSize,
	1388	const void* cSrc, size_t cSrcSize,
	1389	const HUF_DTable* DTable) {
	1390	void const* dt = DTable + 1;
	1391	const BYTE* const iend = (const BYTE*)cSrc + 6;
	1392	BYTE* const oend = (BYTE*)dst + dstSize;
	1393	HUF_DecompressAsmArgs args;
	1394	{
	1395	size_t const ret = HUF_DecompressAsmArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
	1396	FORWARD_IF_ERROR(ret, "Failed to init asm args");
	1397	if (ret != 0)
	1398	return HUF_decompress4X2_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
	1399	}
	1400
	1401	assert(args.ip[0] >= args.ilimit);
	1402	HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop(&args);
	1403
	1404	/* note : op4 already verified within main loop */
	1405	assert(args.ip[0] >= iend);
	1406	assert(args.ip[1] >= iend);
	1407	assert(args.ip[2] >= iend);
	1408	assert(args.ip[3] >= iend);
	1409	assert(args.op[3] <= oend);
	1410	(void)iend;
	1411
	1412	/* finish bitStreams one by one */
	1413	{
	1414	size_t const segmentSize = (dstSize+3) / 4;
	1415	BYTE* segmentEnd = (BYTE*)dst;
	1416	int i;
	1417	for (i = 0; i < 4; ++i) {
	1418	BIT_DStream_t bit;
	1419	if (segmentSize <= (size_t)(oend - segmentEnd))
	1420	segmentEnd += segmentSize;
	1421	else
	1422	segmentEnd = oend;
	1423	FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption");
	1424	args.op[i] += HUF_decodeStreamX2(args.op[i], &bit, segmentEnd, (HUF_DEltX2 const*)dt, HUF_DECODER_FAST_TABLELOG);
	1425	if (args.op[i] != segmentEnd)
	1426	return ERROR(corruption_detected);
	1427	}
	1428	}
	1429
	1430	/* decoded size */
	1431	return dstSize;
	1432	}
	1433	#endif /* ZSTD_ENABLE_ASM_X86_64_BMI2 */
	1434
	1435	static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc,
	1436	size_t cSrcSize, HUF_DTable const* DTable, int bmi2)
	1437	{
	1438	#if DYNAMIC_BMI2
	1439	if (bmi2) {
	1440	# if ZSTD_ENABLE_ASM_X86_64_BMI2
	1441	return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
	1442	# else
	1443	return HUF_decompress4X2_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
	1444	# endif
	1445	}
	1446	#else
	1447	(void)bmi2;
	1448	#endif
	1449
	1450	#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
	1451	return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
	1452	#else
	1453	return HUF_decompress4X2_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable);
	1454	#endif
	1455	}
	1456
	1457	HUF_DGEN(HUF_decompress1X2_usingDTable_internal)
	1458
	1459	size_t HUF_decompress1X2_usingDTable(
	1460	void* dst, size_t dstSize,
	1461	const void* cSrc, size_t cSrcSize,
	1462	const HUF_DTable* DTable)
	1463	{
	1464	DTableDesc dtd = HUF_getDTableDesc(DTable);
	1465	if (dtd.tableType != 1) return ERROR(GENERIC);
	1466	return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
	1467	}
	1468
	1469	size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize,
	1470	const void* cSrc, size_t cSrcSize,
	1471	void* workSpace, size_t wkspSize)
	1472	{
	1473	const BYTE* ip = (const BYTE*) cSrc;
	1474
	1475	size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize,
	1476	workSpace, wkspSize);
	1477	if (HUF_isError(hSize)) return hSize;
	1478	if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
	1479	ip += hSize; cSrcSize -= hSize;
	1480
	1481	return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0);
	1482	}
	1483
	1484
	1485	size_t HUF_decompress4X2_usingDTable(
	1486	void* dst, size_t dstSize,
	1487	const void* cSrc, size_t cSrcSize,
	1488	const HUF_DTable* DTable)
	1489	{
	1490	DTableDesc dtd = HUF_getDTableDesc(DTable);
	1491	if (dtd.tableType != 1) return ERROR(GENERIC);
	1492	return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
	1493	}
	1494
	1495	static size_t HUF_decompress4X2_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize,
	1496	const void* cSrc, size_t cSrcSize,
	1497	void* workSpace, size_t wkspSize, int bmi2)
	1498	{
	1499	const BYTE* ip = (const BYTE*) cSrc;
	1500
	1501	size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize,
	1502	workSpace, wkspSize);
	1503	if (HUF_isError(hSize)) return hSize;
	1504	if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
	1505	ip += hSize; cSrcSize -= hSize;
	1506
	1507	return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2);
	1508	}
	1509
	1510	size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
	1511	const void* cSrc, size_t cSrcSize,
	1512	void* workSpace, size_t wkspSize)
	1513	{
	1514	return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, /* bmi2 */ 0);
	1515	}
	1516
	1517
	1518	#endif /* HUF_FORCE_DECOMPRESS_X1 */
	1519
	1520
	1521	/* ***********************************/
	1522	/* Universal decompression selectors */
	1523	/* ***********************************/
	1524
	1525	size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize,
	1526	const void* cSrc, size_t cSrcSize,
	1527	const HUF_DTable* DTable)
	1528	{
	1529	DTableDesc const dtd = HUF_getDTableDesc(DTable);
	1530	#if defined(HUF_FORCE_DECOMPRESS_X1)
	1531	(void)dtd;
	1532	assert(dtd.tableType == 0);
	1533	return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
	1534	#elif defined(HUF_FORCE_DECOMPRESS_X2)
	1535	(void)dtd;
	1536	assert(dtd.tableType == 1);
	1537	return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
	1538	#else
	1539	return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) :
	1540	HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
	1541	#endif
	1542	}
	1543
	1544	size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize,
	1545	const void* cSrc, size_t cSrcSize,
	1546	const HUF_DTable* DTable)
	1547	{
	1548	DTableDesc const dtd = HUF_getDTableDesc(DTable);
	1549	#if defined(HUF_FORCE_DECOMPRESS_X1)
	1550	(void)dtd;
	1551	assert(dtd.tableType == 0);
	1552	return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
	1553	#elif defined(HUF_FORCE_DECOMPRESS_X2)
	1554	(void)dtd;
	1555	assert(dtd.tableType == 1);
	1556	return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
	1557	#else
	1558	return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) :
	1559	HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
	1560	#endif
	1561	}
	1562
	1563
	1564	#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2)
	1565	typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t;
	1566	static const algo_time_t algoTime[16 /* Quantization /][2 / single, double */] =
	1567	{
	1568	/* single, double, quad */
	1569	{{0,0}, {1,1}}, /* Q==0 : impossible */
	1570	{{0,0}, {1,1}}, /* Q==1 : impossible */
	1571	{{ 150,216}, { 381,119}}, /* Q == 2 : 12-18% */
	1572	{{ 170,205}, { 514,112}}, /* Q == 3 : 18-25% */
	1573	{{ 177,199}, { 539,110}}, /* Q == 4 : 25-32% */
	1574	{{ 197,194}, { 644,107}}, /* Q == 5 : 32-38% */
	1575	{{ 221,192}, { 735,107}}, /* Q == 6 : 38-44% */
	1576	{{ 256,189}, { 881,106}}, /* Q == 7 : 44-50% */
	1577	{{ 359,188}, {1167,109}}, /* Q == 8 : 50-56% */
	1578	{{ 582,187}, {1570,114}}, /* Q == 9 : 56-62% */
	1579	{{ 688,187}, {1712,122}}, /* Q ==10 : 62-69% */
	1580	{{ 825,186}, {1965,136}}, /* Q ==11 : 69-75% */
	1581	{{ 976,185}, {2131,150}}, /* Q ==12 : 75-81% */
	1582	{{1180,186}, {2070,175}}, /* Q ==13 : 81-87% */
	1583	{{1377,185}, {1731,202}}, /* Q ==14 : 87-93% */
	1584	{{1412,185}, {1695,202}}, /* Q ==15 : 93-99% */
	1585	};
	1586	#endif
	1587
	1588	/** HUF_selectDecoder() :
	1589	* Tells which decoder is likely to decode faster,
	1590	* based on a set of pre-computed metrics.
	1591	* @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 .
	1592	* Assumption : 0 < dstSize <= 128 KB */
	1593	U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
	1594	{
	1595	assert(dstSize > 0);
	1596	assert(dstSize <= 128*1024);
	1597	#if defined(HUF_FORCE_DECOMPRESS_X1)
	1598	(void)dstSize;
	1599	(void)cSrcSize;
	1600	return 0;
	1601	#elif defined(HUF_FORCE_DECOMPRESS_X2)
	1602	(void)dstSize;
	1603	(void)cSrcSize;
	1604	return 1;
	1605	#else
	1606	/* decoder timing evaluation */
	1607	{ U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */
	1608	U32 const D256 = (U32)(dstSize >> 8);
	1609	U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256);
	1610	U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
	1611	DTime1 += DTime1 >> 5; /* small advantage to algorithm using less memory, to reduce cache eviction */
	1612	return DTime1 < DTime0;
	1613	}
	1614	#endif
	1615	}
	1616
	1617
	1618	size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst,
	1619	size_t dstSize, const void* cSrc,
	1620	size_t cSrcSize, void* workSpace,
	1621	size_t wkspSize)
	1622	{
	1623	/* validation checks */
	1624	if (dstSize == 0) return ERROR(dstSize_tooSmall);
	1625	if (cSrcSize == 0) return ERROR(corruption_detected);
	1626
	1627	{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
	1628	#if defined(HUF_FORCE_DECOMPRESS_X1)
	1629	(void)algoNb;
	1630	assert(algoNb == 0);
	1631	return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
	1632	#elif defined(HUF_FORCE_DECOMPRESS_X2)
	1633	(void)algoNb;
	1634	assert(algoNb == 1);
	1635	return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
	1636	#else
	1637	return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
	1638	cSrcSize, workSpace, wkspSize):
	1639	HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
	1640	#endif
	1641	}
	1642	}
	1643
	1644	size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
	1645	const void* cSrc, size_t cSrcSize,
	1646	void* workSpace, size_t wkspSize)
	1647	{
	1648	/* validation checks */
	1649	if (dstSize == 0) return ERROR(dstSize_tooSmall);
	1650	if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
	1651	if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
	1652	if (cSrcSize == 1) { ZSTD_memset(dst, (const BYTE)cSrc, dstSize); return dstSize; } /* RLE */
	1653
	1654	{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
	1655	#if defined(HUF_FORCE_DECOMPRESS_X1)
	1656	(void)algoNb;
	1657	assert(algoNb == 0);
	1658	return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
	1659	cSrcSize, workSpace, wkspSize);
	1660	#elif defined(HUF_FORCE_DECOMPRESS_X2)
	1661	(void)algoNb;
	1662	assert(algoNb == 1);
	1663	return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
	1664	cSrcSize, workSpace, wkspSize);
	1665	#else
	1666	return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
	1667	cSrcSize, workSpace, wkspSize):
	1668	HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
	1669	cSrcSize, workSpace, wkspSize);
	1670	#endif
	1671	}
	1672	}
	1673
	1674
	1675	size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2)
	1676	{
	1677	DTableDesc const dtd = HUF_getDTableDesc(DTable);
	1678	#if defined(HUF_FORCE_DECOMPRESS_X1)
	1679	(void)dtd;
	1680	assert(dtd.tableType == 0);
	1681	return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
	1682	#elif defined(HUF_FORCE_DECOMPRESS_X2)
	1683	(void)dtd;
	1684	assert(dtd.tableType == 1);
	1685	return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
	1686	#else
	1687	return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) :
	1688	HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
	1689	#endif
	1690	}
	1691
	1692	#ifndef HUF_FORCE_DECOMPRESS_X2
	1693	size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2)
	1694	{
	1695	const BYTE* ip = (const BYTE*) cSrc;
	1696
	1697	size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
	1698	if (HUF_isError(hSize)) return hSize;
	1699	if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
	1700	ip += hSize; cSrcSize -= hSize;
	1701
	1702	return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2);
	1703	}
	1704	#endif
	1705
	1706	size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2)
	1707	{
	1708	DTableDesc const dtd = HUF_getDTableDesc(DTable);
	1709	#if defined(HUF_FORCE_DECOMPRESS_X1)
	1710	(void)dtd;
	1711	assert(dtd.tableType == 0);
	1712	return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
	1713	#elif defined(HUF_FORCE_DECOMPRESS_X2)
	1714	(void)dtd;
	1715	assert(dtd.tableType == 1);
	1716	return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
	1717	#else
	1718	return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) :
	1719	HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
	1720	#endif
	1721	}
	1722
	1723	size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2)
	1724	{
	1725	/* validation checks */
	1726	if (dstSize == 0) return ERROR(dstSize_tooSmall);
	1727	if (cSrcSize == 0) return ERROR(corruption_detected);
	1728
	1729	{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
	1730	#if defined(HUF_FORCE_DECOMPRESS_X1)
	1731	(void)algoNb;
	1732	assert(algoNb == 0);
	1733	return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
	1734	#elif defined(HUF_FORCE_DECOMPRESS_X2)
	1735	(void)algoNb;
	1736	assert(algoNb == 1);
	1737	return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
	1738	#else
	1739	return algoNb ? HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2) :
	1740	HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
	1741	#endif
	1742	}
	1743	}
	1744
	1745	#ifndef ZSTD_NO_UNUSED_FUNCTIONS
	1746	#ifndef HUF_FORCE_DECOMPRESS_X2
	1747	size_t HUF_readDTableX1(HUF_DTable* DTable, const void* src, size_t srcSize)
	1748	{
	1749	U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
	1750	return HUF_readDTableX1_wksp(DTable, src, srcSize,
	1751	workSpace, sizeof(workSpace));
	1752	}
	1753
	1754	size_t HUF_decompress1X1_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize,
	1755	const void* cSrc, size_t cSrcSize)
	1756	{
	1757	U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
	1758	return HUF_decompress1X1_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize,
	1759	workSpace, sizeof(workSpace));
	1760	}
	1761
	1762	size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
	1763	{
	1764	HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX);
	1765	return HUF_decompress1X1_DCtx (DTable, dst, dstSize, cSrc, cSrcSize);
	1766	}
	1767	#endif
	1768
	1769	#ifndef HUF_FORCE_DECOMPRESS_X1
	1770	size_t HUF_readDTableX2(HUF_DTable* DTable, const void* src, size_t srcSize)
	1771	{
	1772	U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
	1773	return HUF_readDTableX2_wksp(DTable, src, srcSize,
	1774	workSpace, sizeof(workSpace));
	1775	}
	1776
	1777	size_t HUF_decompress1X2_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize,
	1778	const void* cSrc, size_t cSrcSize)
	1779	{
	1780	U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
	1781	return HUF_decompress1X2_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize,
	1782	workSpace, sizeof(workSpace));
	1783	}
	1784
	1785	size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
	1786	{
	1787	HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);
	1788	return HUF_decompress1X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
	1789	}
	1790	#endif
	1791
	1792	#ifndef HUF_FORCE_DECOMPRESS_X2
	1793	size_t HUF_decompress4X1_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
	1794	{
	1795	U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
	1796	return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
	1797	workSpace, sizeof(workSpace));
	1798	}
	1799	size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
	1800	{
	1801	HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX);
	1802	return HUF_decompress4X1_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
	1803	}
	1804	#endif
	1805
	1806	#ifndef HUF_FORCE_DECOMPRESS_X1
	1807	size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize,
	1808	const void* cSrc, size_t cSrcSize)
	1809	{
	1810	U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
	1811	return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
	1812	workSpace, sizeof(workSpace));
	1813	}
	1814
	1815	size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
	1816	{
	1817	HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);
	1818	return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
	1819	}
	1820	#endif
	1821
	1822	typedef size_t (decompressionAlgo)(void dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
	1823
	1824	size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
	1825	{
	1826	#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2)
	1827	static const decompressionAlgo decompress[2] = { HUF_decompress4X1, HUF_decompress4X2 };
	1828	#endif
	1829
	1830	/* validation checks */
	1831	if (dstSize == 0) return ERROR(dstSize_tooSmall);
	1832	if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
	1833	if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
	1834	if (cSrcSize == 1) { ZSTD_memset(dst, (const BYTE)cSrc, dstSize); return dstSize; } /* RLE */
	1835
	1836	{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
	1837	#if defined(HUF_FORCE_DECOMPRESS_X1)
	1838	(void)algoNb;
	1839	assert(algoNb == 0);
	1840	return HUF_decompress4X1(dst, dstSize, cSrc, cSrcSize);
	1841	#elif defined(HUF_FORCE_DECOMPRESS_X2)
	1842	(void)algoNb;
	1843	assert(algoNb == 1);
	1844	return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize);
	1845	#else
	1846	return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
	1847	#endif
	1848	}
	1849	}
	1850
	1851	size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
	1852	{
	1853	/* validation checks */
	1854	if (dstSize == 0) return ERROR(dstSize_tooSmall);
	1855	if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
	1856	if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
	1857	if (cSrcSize == 1) { ZSTD_memset(dst, (const BYTE)cSrc, dstSize); return dstSize; } /* RLE */
	1858
	1859	{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
	1860	#if defined(HUF_FORCE_DECOMPRESS_X1)
	1861	(void)algoNb;
	1862	assert(algoNb == 0);
	1863	return HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize);
	1864	#elif defined(HUF_FORCE_DECOMPRESS_X2)
	1865	(void)algoNb;
	1866	assert(algoNb == 1);
	1867	return HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize);
	1868	#else
	1869	return algoNb ? HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
	1870	HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
	1871	#endif
	1872	}
	1873	}
	1874
	1875	size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
	1876	{
	1877	U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
	1878	return HUF_decompress4X_hufOnly_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
	1879	workSpace, sizeof(workSpace));
	1880	}
	1881
	1882	size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize,
	1883	const void* cSrc, size_t cSrcSize)
	1884	{
	1885	U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32];
	1886	return HUF_decompress1X_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize,
	1887	workSpace, sizeof(workSpace));
	1888	}
	1889	#endif

stage1/zstd/lib/decompress/huf_decompress_amd64.S created+585

...	...	@@ -0,0 +1,585 @@
	1	/*
	2	* Copyright (c) Facebook, Inc.
	3	* All rights reserved.
	4	*
	5	* This source code is licensed under both the BSD-style license (found in the
	6	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	7	* in the COPYING file in the root directory of this source tree).
	8	* You may select, at your option, one of the above-listed licenses.
	9	*/
	10
	11	#include "../common/portability_macros.h"
	12
	13	/* Stack marking
	14	* ref: https://wiki.gentoo.org/wiki/Hardened/GNU_stack_quickstart
	15	*/
	16	#if defined(__ELF__) && defined(__GNUC__)
	17	.section .note.GNU-stack,"",%progbits
	18	#endif
	19
	20	#if ZSTD_ENABLE_ASM_X86_64_BMI2
	21
	22	/* Calling convention:
	23	*
	24	* %rdi contains the first argument: HUF_DecompressAsmArgs*.
	25	* %rbp isn't maintained (no frame pointer).
	26	* %rsp contains the stack pointer that grows down.
	27	* No red-zone is assumed, only addresses >= %rsp are used.
	28	* All register contents are preserved.
	29	*
	30	* TODO: Support Windows calling convention.
	31	*/
	32
	33	ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop)
	34	ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop)
	35	ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop)
	36	ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop)
	37	.global HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop
	38	.global HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop
	39	.global _HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop
	40	.global _HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop
	41	.text
	42
	43	/* Sets up register mappings for clarity.
	44	* op[], bits[], dtable & ip[0] each get their own register.
	45	* ip[1,2,3] & olimit alias var[].
	46	* %rax is a scratch register.
	47	*/
	48
	49	#define op0 rsi
	50	#define op1 rbx
	51	#define op2 rcx
	52	#define op3 rdi
	53
	54	#define ip0 r8
	55	#define ip1 r9
	56	#define ip2 r10
	57	#define ip3 r11
	58
	59	#define bits0 rbp
	60	#define bits1 rdx
	61	#define bits2 r12
	62	#define bits3 r13
	63	#define dtable r14
	64	#define olimit r15
	65
	66	/* var[] aliases ip[1,2,3] & olimit
	67	* ip[1,2,3] are saved every iteration.
	68	* olimit is only used in compute_olimit.
	69	*/
	70	#define var0 r15
	71	#define var1 r9
	72	#define var2 r10
	73	#define var3 r11
	74
	75	/* 32-bit var registers */
	76	#define vard0 r15d
	77	#define vard1 r9d
	78	#define vard2 r10d
	79	#define vard3 r11d
	80
	81	/* Calls X(N) for each stream 0, 1, 2, 3. */
	82	#define FOR_EACH_STREAM(X) \
	83	X(0); \
	84	X(1); \
	85	X(2); \
	86	X(3)
	87
	88	/* Calls X(N, idx) for each stream 0, 1, 2, 3. */
	89	#define FOR_EACH_STREAM_WITH_INDEX(X, idx) \
	90	X(0, idx); \
	91	X(1, idx); \
	92	X(2, idx); \
	93	X(3, idx)
	94
	95	/* Define both _HUF_* & HUF_* symbols because MacOS
	96	* C symbols are prefixed with '_' & Linux symbols aren't.
	97	*/
	98	_HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop:
	99	HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop:
	100	/* Save all registers - even if they are callee saved for simplicity. */
	101	push %rax
	102	push %rbx
	103	push %rcx
	104	push %rdx
	105	push %rbp
	106	push %rsi
	107	push %rdi
	108	push %r8
	109	push %r9
	110	push %r10
	111	push %r11
	112	push %r12
	113	push %r13
	114	push %r14
	115	push %r15
	116
	117	/* Read HUF_DecompressAsmArgs* args from %rax */
	118	movq %rdi, %rax
	119	movq 0(%rax), %ip0
	120	movq 8(%rax), %ip1
	121	movq 16(%rax), %ip2
	122	movq 24(%rax), %ip3
	123	movq 32(%rax), %op0
	124	movq 40(%rax), %op1
	125	movq 48(%rax), %op2
	126	movq 56(%rax), %op3
	127	movq 64(%rax), %bits0
	128	movq 72(%rax), %bits1
	129	movq 80(%rax), %bits2
	130	movq 88(%rax), %bits3
	131	movq 96(%rax), %dtable
	132	push %rax /* argument */
	133	push 104(%rax) /* ilimit */
	134	push 112(%rax) /* oend */
	135	push %olimit /* olimit space */
	136
	137	subq $24, %rsp
	138
	139	.L_4X1_compute_olimit:
	140	/* Computes how many iterations we can do safely
	141	* %r15, %rax may be clobbered
	142	* rbx, rdx must be saved
	143	* op3 & ip0 mustn't be clobbered
	144	*/
	145	movq %rbx, 0(%rsp)
	146	movq %rdx, 8(%rsp)
	147
	148	movq 32(%rsp), %rax /* rax = oend */
	149	subq %op3, %rax /* rax = oend - op3 */
	150
	151	/* r15 = (oend - op3) / 5 */
	152	movabsq $-3689348814741910323, %rdx
	153	mulq %rdx
	154	movq %rdx, %r15
	155	shrq $2, %r15
	156
	157	movq %ip0, %rax /* rax = ip0 */
	158	movq 40(%rsp), %rdx /* rdx = ilimit */
	159	subq %rdx, %rax /* rax = ip0 - ilimit */
	160	movq %rax, %rbx /* rbx = ip0 - ilimit */
	161
	162	/* rdx = (ip0 - ilimit) / 7 */
	163	movabsq $2635249153387078803, %rdx
	164	mulq %rdx
	165	subq %rdx, %rbx
	166	shrq %rbx
	167	addq %rbx, %rdx
	168	shrq $2, %rdx
	169
	170	/* r15 = min(%rdx, %r15) */
	171	cmpq %rdx, %r15
	172	cmova %rdx, %r15
	173
	174	/* r15 = r15 * 5 */
	175	leaq (%r15, %r15, 4), %r15
	176
	177	/* olimit = op3 + r15 */
	178	addq %op3, %olimit
	179
	180	movq 8(%rsp), %rdx
	181	movq 0(%rsp), %rbx
	182
	183	/* If (op3 + 20 > olimit) */
	184	movq %op3, %rax /* rax = op3 */
	185	addq $20, %rax /* rax = op3 + 20 */
	186	cmpq %rax, %olimit /* op3 + 20 > olimit */
	187	jb .L_4X1_exit
	188
	189	/* If (ip1 < ip0) go to exit */
	190	cmpq %ip0, %ip1
	191	jb .L_4X1_exit
	192
	193	/* If (ip2 < ip1) go to exit */
	194	cmpq %ip1, %ip2
	195	jb .L_4X1_exit
	196
	197	/* If (ip3 < ip2) go to exit */
	198	cmpq %ip2, %ip3
	199	jb .L_4X1_exit
	200
	201	/* Reads top 11 bits from bits[n]
	202	* Loads dt[bits[n]] into var[n]
	203	*/
	204	#define GET_NEXT_DELT(n) \
	205	movq $53, %var##n; \
	206	shrxq %var##n, %bits##n, %var##n; \
	207	movzwl (%dtable,%var##n,2),%vard##n
	208
	209	/* var[n] must contain the DTable entry computed with GET_NEXT_DELT
	210	* Moves var[n] to %rax
	211	* bits[n] <<= var[n] & 63
	212	* op[n][idx] = %rax >> 8
	213	* %ah is a way to access bits [8, 16) of %rax
	214	*/
	215	#define DECODE_FROM_DELT(n, idx) \
	216	movq %var##n, %rax; \
	217	shlxq %var##n, %bits##n, %bits##n; \
	218	movb %ah, idx(%op##n)
	219
	220	/* Assumes GET_NEXT_DELT has been called.
	221	* Calls DECODE_FROM_DELT then GET_NEXT_DELT
	222	*/
	223	#define DECODE_AND_GET_NEXT(n, idx) \
	224	DECODE_FROM_DELT(n, idx); \
	225	GET_NEXT_DELT(n) \
	226
	227	/* // ctz & nbBytes is stored in bits[n]
	228	* // nbBits is stored in %rax
	229	* ctz = CTZ[bits[n]]
	230	* nbBits = ctz & 7
	231	* nbBytes = ctz >> 3
	232	* op[n] += 5
	233	* ip[n] -= nbBytes
	234	* // Note: x86-64 is little-endian ==> no bswap
	235	* bits[n] = MEM_readST(ip[n]) \| 1
	236	* bits[n] <<= nbBits
	237	*/
	238	#define RELOAD_BITS(n) \
	239	bsfq %bits##n, %bits##n; \
	240	movq %bits##n, %rax; \
	241	andq $7, %rax; \
	242	shrq $3, %bits##n; \
	243	leaq 5(%op##n), %op##n; \
	244	subq %bits##n, %ip##n; \
	245	movq (%ip##n), %bits##n; \
	246	orq $1, %bits##n; \
	247	shlx %rax, %bits##n, %bits##n
	248
	249	/* Store clobbered variables on the stack */
	250	movq %olimit, 24(%rsp)
	251	movq %ip1, 0(%rsp)
	252	movq %ip2, 8(%rsp)
	253	movq %ip3, 16(%rsp)
	254
	255	/* Call GET_NEXT_DELT for each stream */
	256	FOR_EACH_STREAM(GET_NEXT_DELT)
	257
	258	.p2align 6
	259
	260	.L_4X1_loop_body:
	261	/* Decode 5 symbols in each of the 4 streams (20 total)
	262	* Must have called GET_NEXT_DELT for each stream
	263	*/
	264	FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 0)
	265	FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 1)
	266	FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 2)
	267	FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 3)
	268	FOR_EACH_STREAM_WITH_INDEX(DECODE_FROM_DELT, 4)
	269
	270	/* Load ip[1,2,3] from stack (var[] aliases them)
	271	* ip[] is needed for RELOAD_BITS
	272	* Each will be stored back to the stack after RELOAD
	273	*/
	274	movq 0(%rsp), %ip1
	275	movq 8(%rsp), %ip2
	276	movq 16(%rsp), %ip3
	277
	278	/* Reload each stream & fetch the next table entry
	279	* to prepare for the next iteration
	280	*/
	281	RELOAD_BITS(0)
	282	GET_NEXT_DELT(0)
	283
	284	RELOAD_BITS(1)
	285	movq %ip1, 0(%rsp)
	286	GET_NEXT_DELT(1)
	287
	288	RELOAD_BITS(2)
	289	movq %ip2, 8(%rsp)
	290	GET_NEXT_DELT(2)
	291
	292	RELOAD_BITS(3)
	293	movq %ip3, 16(%rsp)
	294	GET_NEXT_DELT(3)
	295
	296	/* If op3 < olimit: continue the loop */
	297	cmp %op3, 24(%rsp)
	298	ja .L_4X1_loop_body
	299
	300	/* Reload ip[1,2,3] from stack */
	301	movq 0(%rsp), %ip1
	302	movq 8(%rsp), %ip2
	303	movq 16(%rsp), %ip3
	304
	305	/* Re-compute olimit */
	306	jmp .L_4X1_compute_olimit
	307
	308	#undef GET_NEXT_DELT
	309	#undef DECODE_FROM_DELT
	310	#undef DECODE
	311	#undef RELOAD_BITS
	312	.L_4X1_exit:
	313	addq $24, %rsp
	314
	315	/* Restore stack (oend & olimit) */
	316	pop %rax /* olimit */
	317	pop %rax /* oend */
	318	pop %rax /* ilimit */
	319	pop %rax /* arg */
	320
	321	/* Save ip / op / bits */
	322	movq %ip0, 0(%rax)
	323	movq %ip1, 8(%rax)
	324	movq %ip2, 16(%rax)
	325	movq %ip3, 24(%rax)
	326	movq %op0, 32(%rax)
	327	movq %op1, 40(%rax)
	328	movq %op2, 48(%rax)
	329	movq %op3, 56(%rax)
	330	movq %bits0, 64(%rax)
	331	movq %bits1, 72(%rax)
	332	movq %bits2, 80(%rax)
	333	movq %bits3, 88(%rax)
	334
	335	/* Restore registers */
	336	pop %r15
	337	pop %r14
	338	pop %r13
	339	pop %r12
	340	pop %r11
	341	pop %r10
	342	pop %r9
	343	pop %r8
	344	pop %rdi
	345	pop %rsi
	346	pop %rbp
	347	pop %rdx
	348	pop %rcx
	349	pop %rbx
	350	pop %rax
	351	ret
	352
	353	_HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop:
	354	HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop:
	355	/* Save all registers - even if they are callee saved for simplicity. */
	356	push %rax
	357	push %rbx
	358	push %rcx
	359	push %rdx
	360	push %rbp
	361	push %rsi
	362	push %rdi
	363	push %r8
	364	push %r9
	365	push %r10
	366	push %r11
	367	push %r12
	368	push %r13
	369	push %r14
	370	push %r15
	371
	372	movq %rdi, %rax
	373	movq 0(%rax), %ip0
	374	movq 8(%rax), %ip1
	375	movq 16(%rax), %ip2
	376	movq 24(%rax), %ip3
	377	movq 32(%rax), %op0
	378	movq 40(%rax), %op1
	379	movq 48(%rax), %op2
	380	movq 56(%rax), %op3
	381	movq 64(%rax), %bits0
	382	movq 72(%rax), %bits1
	383	movq 80(%rax), %bits2
	384	movq 88(%rax), %bits3
	385	movq 96(%rax), %dtable
	386	push %rax /* argument */
	387	push %rax /* olimit */
	388	push 104(%rax) /* ilimit */
	389
	390	movq 112(%rax), %rax
	391	push %rax /* oend3 */
	392
	393	movq %op3, %rax
	394	push %rax /* oend2 */
	395
	396	movq %op2, %rax
	397	push %rax /* oend1 */
	398
	399	movq %op1, %rax
	400	push %rax /* oend0 */
	401
	402	/* Scratch space */
	403	subq $8, %rsp
	404
	405	.L_4X2_compute_olimit:
	406	/* Computes how many iterations we can do safely
	407	* %r15, %rax may be clobbered
	408	* rdx must be saved
	409	* op[1,2,3,4] & ip0 mustn't be clobbered
	410	*/
	411	movq %rdx, 0(%rsp)
	412
	413	/* We can consume up to 7 input bytes each iteration. */
	414	movq %ip0, %rax /* rax = ip0 */
	415	movq 40(%rsp), %rdx /* rdx = ilimit */
	416	subq %rdx, %rax /* rax = ip0 - ilimit */
	417	movq %rax, %r15 /* r15 = ip0 - ilimit */
	418
	419	/* rdx = rax / 7 */
	420	movabsq $2635249153387078803, %rdx
	421	mulq %rdx
	422	subq %rdx, %r15
	423	shrq %r15
	424	addq %r15, %rdx
	425	shrq $2, %rdx
	426
	427	/* r15 = (ip0 - ilimit) / 7 */
	428	movq %rdx, %r15
	429
	430	movabsq $-3689348814741910323, %rdx
	431	movq 8(%rsp), %rax /* rax = oend0 */
	432	subq %op0, %rax /* rax = oend0 - op0 */
	433	mulq %rdx
	434	shrq $3, %rdx /* rdx = rax / 10 */
	435
	436	/* r15 = min(%rdx, %r15) */
	437	cmpq %rdx, %r15
	438	cmova %rdx, %r15
	439
	440	movabsq $-3689348814741910323, %rdx
	441	movq 16(%rsp), %rax /* rax = oend1 */
	442	subq %op1, %rax /* rax = oend1 - op1 */
	443	mulq %rdx
	444	shrq $3, %rdx /* rdx = rax / 10 */
	445
	446	/* r15 = min(%rdx, %r15) */
	447	cmpq %rdx, %r15
	448	cmova %rdx, %r15
	449
	450	movabsq $-3689348814741910323, %rdx
	451	movq 24(%rsp), %rax /* rax = oend2 */
	452	subq %op2, %rax /* rax = oend2 - op2 */
	453	mulq %rdx
	454	shrq $3, %rdx /* rdx = rax / 10 */
	455
	456	/* r15 = min(%rdx, %r15) */
	457	cmpq %rdx, %r15
	458	cmova %rdx, %r15
	459
	460	movabsq $-3689348814741910323, %rdx
	461	movq 32(%rsp), %rax /* rax = oend3 */
	462	subq %op3, %rax /* rax = oend3 - op3 */
	463	mulq %rdx
	464	shrq $3, %rdx /* rdx = rax / 10 */
	465
	466	/* r15 = min(%rdx, %r15) */
	467	cmpq %rdx, %r15
	468	cmova %rdx, %r15
	469
	470	/* olimit = op3 + 5 * r15 */
	471	movq %r15, %rax
	472	leaq (%op3, %rax, 4), %olimit
	473	addq %rax, %olimit
	474
	475	movq 0(%rsp), %rdx
	476
	477	/* If (op3 + 10 > olimit) */
	478	movq %op3, %rax /* rax = op3 */
	479	addq $10, %rax /* rax = op3 + 10 */
	480	cmpq %rax, %olimit /* op3 + 10 > olimit */
	481	jb .L_4X2_exit
	482
	483	/* If (ip1 < ip0) go to exit */
	484	cmpq %ip0, %ip1
	485	jb .L_4X2_exit
	486
	487	/* If (ip2 < ip1) go to exit */
	488	cmpq %ip1, %ip2
	489	jb .L_4X2_exit
	490
	491	/* If (ip3 < ip2) go to exit */
	492	cmpq %ip2, %ip3
	493	jb .L_4X2_exit
	494
	495	#define DECODE(n, idx) \
	496	movq %bits##n, %rax; \
	497	shrq $53, %rax; \
	498	movzwl 0(%dtable,%rax,4),%r8d; \
	499	movzbl 2(%dtable,%rax,4),%r15d; \
	500	movzbl 3(%dtable,%rax,4),%eax; \
	501	movw %r8w, (%op##n); \
	502	shlxq %r15, %bits##n, %bits##n; \
	503	addq %rax, %op##n
	504
	505	#define RELOAD_BITS(n) \
	506	bsfq %bits##n, %bits##n; \
	507	movq %bits##n, %rax; \
	508	shrq $3, %bits##n; \
	509	andq $7, %rax; \
	510	subq %bits##n, %ip##n; \
	511	movq (%ip##n), %bits##n; \
	512	orq $1, %bits##n; \
	513	shlxq %rax, %bits##n, %bits##n
	514
	515
	516	movq %olimit, 48(%rsp)
	517
	518	.p2align 6
	519
	520	.L_4X2_loop_body:
	521	/* We clobber r8, so store it on the stack */
	522	movq %r8, 0(%rsp)
	523
	524	/* Decode 5 symbols from each of the 4 streams (20 symbols total). */
	525	FOR_EACH_STREAM_WITH_INDEX(DECODE, 0)
	526	FOR_EACH_STREAM_WITH_INDEX(DECODE, 1)
	527	FOR_EACH_STREAM_WITH_INDEX(DECODE, 2)
	528	FOR_EACH_STREAM_WITH_INDEX(DECODE, 3)
	529	FOR_EACH_STREAM_WITH_INDEX(DECODE, 4)
	530
	531	/* Reload r8 */
	532	movq 0(%rsp), %r8
	533
	534	FOR_EACH_STREAM(RELOAD_BITS)
	535
	536	cmp %op3, 48(%rsp)
	537	ja .L_4X2_loop_body
	538	jmp .L_4X2_compute_olimit
	539
	540	#undef DECODE
	541	#undef RELOAD_BITS
	542	.L_4X2_exit:
	543	addq $8, %rsp
	544	/* Restore stack (oend & olimit) */
	545	pop %rax /* oend0 */
	546	pop %rax /* oend1 */
	547	pop %rax /* oend2 */
	548	pop %rax /* oend3 */
	549	pop %rax /* ilimit */
	550	pop %rax /* olimit */
	551	pop %rax /* arg */
	552
	553	/* Save ip / op / bits */
	554	movq %ip0, 0(%rax)
	555	movq %ip1, 8(%rax)
	556	movq %ip2, 16(%rax)
	557	movq %ip3, 24(%rax)
	558	movq %op0, 32(%rax)
	559	movq %op1, 40(%rax)
	560	movq %op2, 48(%rax)
	561	movq %op3, 56(%rax)
	562	movq %bits0, 64(%rax)
	563	movq %bits1, 72(%rax)
	564	movq %bits2, 80(%rax)
	565	movq %bits3, 88(%rax)
	566
	567	/* Restore registers */
	568	pop %r15
	569	pop %r14
	570	pop %r13
	571	pop %r12
	572	pop %r11
	573	pop %r10
	574	pop %r9
	575	pop %r8
	576	pop %rdi
	577	pop %rsi
	578	pop %rbp
	579	pop %rdx
	580	pop %rcx
	581	pop %rbx
	582	pop %rax
	583	ret
	584
	585	#endif

stage1/zstd/lib/decompress/zstd_ddict.c created+244

...	...	@@ -0,0 +1,244 @@
	1	/*
	2	* Copyright (c) Yann Collet, Facebook, Inc.
	3	* All rights reserved.
	4	*
	5	* This source code is licensed under both the BSD-style license (found in the
	6	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	7	* in the COPYING file in the root directory of this source tree).
	8	* You may select, at your option, one of the above-listed licenses.
	9	*/
	10
	11	/* zstd_ddict.c :
	12	* concentrates all logic that needs to know the internals of ZSTD_DDict object */
	13
	14	/-******************************************************
	15	* Dependencies
	16	*********************************************************/
	17	#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
	18	#include "../common/cpu.h" /* bmi2 */
	19	#include "../common/mem.h" /* low level memory routines */
	20	#define FSE_STATIC_LINKING_ONLY
	21	#include "../common/fse.h"
	22	#define HUF_STATIC_LINKING_ONLY
	23	#include "../common/huf.h"
	24	#include "zstd_decompress_internal.h"
	25	#include "zstd_ddict.h"
	26
	27	#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
	28	# include "../legacy/zstd_legacy.h"
	29	#endif
	30
	31
	32
	33	/-******************************************************
	34	* Types
	35	*********************************************************/
	36	struct ZSTD_DDict_s {
	37	void* dictBuffer;
	38	const void* dictContent;
	39	size_t dictSize;
	40	ZSTD_entropyDTables_t entropy;
	41	U32 dictID;
	42	U32 entropyPresent;
	43	ZSTD_customMem cMem;
	44	}; /* typedef'd to ZSTD_DDict within "zstd.h" */
	45
	46	const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict)
	47	{
	48	assert(ddict != NULL);
	49	return ddict->dictContent;
	50	}
	51
	52	size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict)
	53	{
	54	assert(ddict != NULL);
	55	return ddict->dictSize;
	56	}
	57
	58	void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
	59	{
	60	DEBUGLOG(4, "ZSTD_copyDDictParameters");
	61	assert(dctx != NULL);
	62	assert(ddict != NULL);
	63	dctx->dictID = ddict->dictID;
	64	dctx->prefixStart = ddict->dictContent;
	65	dctx->virtualStart = ddict->dictContent;
	66	dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize;
	67	dctx->previousDstEnd = dctx->dictEnd;
	68	#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
	69	dctx->dictContentBeginForFuzzing = dctx->prefixStart;
	70	dctx->dictContentEndForFuzzing = dctx->previousDstEnd;
	71	#endif
	72	if (ddict->entropyPresent) {
	73	dctx->litEntropy = 1;
	74	dctx->fseEntropy = 1;
	75	dctx->LLTptr = ddict->entropy.LLTable;
	76	dctx->MLTptr = ddict->entropy.MLTable;
	77	dctx->OFTptr = ddict->entropy.OFTable;
	78	dctx->HUFptr = ddict->entropy.hufTable;
	79	dctx->entropy.rep[0] = ddict->entropy.rep[0];
	80	dctx->entropy.rep[1] = ddict->entropy.rep[1];
	81	dctx->entropy.rep[2] = ddict->entropy.rep[2];
	82	} else {
	83	dctx->litEntropy = 0;
	84	dctx->fseEntropy = 0;
	85	}
	86	}
	87
	88
	89	static size_t
	90	ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict,
	91	ZSTD_dictContentType_e dictContentType)
	92	{
	93	ddict->dictID = 0;
	94	ddict->entropyPresent = 0;
	95	if (dictContentType == ZSTD_dct_rawContent) return 0;
	96
	97	if (ddict->dictSize < 8) {
	98	if (dictContentType == ZSTD_dct_fullDict)
	99	return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
	100	return 0; /* pure content mode */
	101	}
	102	{ U32 const magic = MEM_readLE32(ddict->dictContent);
	103	if (magic != ZSTD_MAGIC_DICTIONARY) {
	104	if (dictContentType == ZSTD_dct_fullDict)
	105	return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
	106	return 0; /* pure content mode */
	107	}
	108	}
	109	ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE);
	110
	111	/* load entropy tables */
	112	RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy(
	113	&ddict->entropy, ddict->dictContent, ddict->dictSize)),
	114	dictionary_corrupted, "");
	115	ddict->entropyPresent = 1;
	116	return 0;
	117	}
	118
	119
	120	static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
	121	const void* dict, size_t dictSize,
	122	ZSTD_dictLoadMethod_e dictLoadMethod,
	123	ZSTD_dictContentType_e dictContentType)
	124	{
	125	if ((dictLoadMethod == ZSTD_dlm_byRef) \|\| (!dict) \|\| (!dictSize)) {
	126	ddict->dictBuffer = NULL;
	127	ddict->dictContent = dict;
	128	if (!dict) dictSize = 0;
	129	} else {
	130	void* const internalBuffer = ZSTD_customMalloc(dictSize, ddict->cMem);
	131	ddict->dictBuffer = internalBuffer;
	132	ddict->dictContent = internalBuffer;
	133	if (!internalBuffer) return ERROR(memory_allocation);
	134	ZSTD_memcpy(internalBuffer, dict, dictSize);
	135	}
	136	ddict->dictSize = dictSize;
	137	ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)0x1000001); / cover both little and big endian */
	138
	139	/* parse dictionary content */
	140	FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , "");
	141
	142	return 0;
	143	}
	144
	145	ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
	146	ZSTD_dictLoadMethod_e dictLoadMethod,
	147	ZSTD_dictContentType_e dictContentType,
	148	ZSTD_customMem customMem)
	149	{
	150	if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
	151
	152	{ ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_customMalloc(sizeof(ZSTD_DDict), customMem);
	153	if (ddict == NULL) return NULL;
	154	ddict->cMem = customMem;
	155	{ size_t const initResult = ZSTD_initDDict_internal(ddict,
	156	dict, dictSize,
	157	dictLoadMethod, dictContentType);
	158	if (ZSTD_isError(initResult)) {
	159	ZSTD_freeDDict(ddict);
	160	return NULL;
	161	} }
	162	return ddict;
	163	}
	164	}
	165
	166	/*! ZSTD_createDDict() :
	167	* Create a digested dictionary, to start decompression without startup delay.
	168	* `dict` content is copied inside DDict.
	169	* Consequently, `dict` can be released after `ZSTD_DDict` creation */
	170	ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
	171	{
	172	ZSTD_customMem const allocator = { NULL, NULL, NULL };
	173	return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator);
	174	}
	175
	176	/*! ZSTD_createDDict_byReference() :
	177	* Create a digested dictionary, to start decompression without startup delay.
	178	* Dictionary content is simply referenced, it will be accessed during decompression.
	179	* Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */
	180	ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
	181	{
	182	ZSTD_customMem const allocator = { NULL, NULL, NULL };
	183	return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator);
	184	}
	185
	186
	187	const ZSTD_DDict* ZSTD_initStaticDDict(
	188	void* sBuffer, size_t sBufferSize,
	189	const void* dict, size_t dictSize,
	190	ZSTD_dictLoadMethod_e dictLoadMethod,
	191	ZSTD_dictContentType_e dictContentType)
	192	{
	193	size_t const neededSpace = sizeof(ZSTD_DDict)
	194	+ (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
	195	ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer;
	196	assert(sBuffer != NULL);
	197	assert(dict != NULL);
	198	if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */
	199	if (sBufferSize < neededSpace) return NULL;
	200	if (dictLoadMethod == ZSTD_dlm_byCopy) {
	201	ZSTD_memcpy(ddict+1, dict, dictSize); /* local copy */
	202	dict = ddict+1;
	203	}
	204	if (ZSTD_isError( ZSTD_initDDict_internal(ddict,
	205	dict, dictSize,
	206	ZSTD_dlm_byRef, dictContentType) ))
	207	return NULL;
	208	return ddict;
	209	}
	210
	211
	212	size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
	213	{
	214	if (ddict==NULL) return 0; /* support free on NULL */
	215	{ ZSTD_customMem const cMem = ddict->cMem;
	216	ZSTD_customFree(ddict->dictBuffer, cMem);
	217	ZSTD_customFree(ddict, cMem);
	218	return 0;
	219	}
	220	}
	221
	222	/*! ZSTD_estimateDDictSize() :
	223	* Estimate amount of memory that will be needed to create a dictionary for decompression.
	224	* Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */
	225	size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod)
	226	{
	227	return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
	228	}
	229
	230	size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
	231	{
	232	if (ddict==NULL) return 0; /* support sizeof on NULL */
	233	return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ;
	234	}
	235
	236	/*! ZSTD_getDictID_fromDDict() :
	237	* Provides the dictID of the dictionary loaded into `ddict`.
	238	* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
	239	* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
	240	unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
	241	{
	242	if (ddict==NULL) return 0;
	243	return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
	244	}

stage1/zstd/lib/decompress/zstd_ddict.h created+44

...	...	@@ -0,0 +1,44 @@
	1	/*
	2	* Copyright (c) Yann Collet, Facebook, Inc.
	3	* All rights reserved.
	4	*
	5	* This source code is licensed under both the BSD-style license (found in the
	6	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	7	* in the COPYING file in the root directory of this source tree).
	8	* You may select, at your option, one of the above-listed licenses.
	9	*/
	10
	11
	12	#ifndef ZSTD_DDICT_H
	13	#define ZSTD_DDICT_H
	14
	15	/-******************************************************
	16	* Dependencies
	17	*********************************************************/
	18	#include "../common/zstd_deps.h" /* size_t */
	19	#include "../zstd.h" /* ZSTD_DDict, and several public functions */
	20
	21
	22	/-******************************************************
	23	* Interface
	24	*********************************************************/
	25
	26	/* note: several prototypes are already published in `zstd.h` :
	27	* ZSTD_createDDict()
	28	* ZSTD_createDDict_byReference()
	29	* ZSTD_createDDict_advanced()
	30	* ZSTD_freeDDict()
	31	* ZSTD_initStaticDDict()
	32	* ZSTD_sizeof_DDict()
	33	* ZSTD_estimateDDictSize()
	34	* ZSTD_getDictID_fromDict()
	35	*/
	36
	37	const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict);
	38	size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict);
	39
	40	void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
	41
	42
	43
	44	#endif /* ZSTD_DDICT_H */

stage1/zstd/lib/decompress/zstd_decompress.c created+2230

...	...	@@ -0,0 +1,2230 @@
	1	/*
	2	* Copyright (c) Yann Collet, Facebook, Inc.
	3	* All rights reserved.
	4	*
	5	* This source code is licensed under both the BSD-style license (found in the
	6	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	7	* in the COPYING file in the root directory of this source tree).
	8	* You may select, at your option, one of the above-listed licenses.
	9	*/
	10
	11
	12	/* ***************************************************************
	13	* Tuning parameters
	14	*****************************************************************/
	15	/*!
	16	* HEAPMODE :
	17	* Select how default decompression function ZSTD_decompress() allocates its context,
	18	* on stack (0), or into heap (1, default; requires malloc()).
	19	* Note that functions with explicit context such as ZSTD_decompressDCtx() are unaffected.
	20	*/
	21	#ifndef ZSTD_HEAPMODE
	22	# define ZSTD_HEAPMODE 1
	23	#endif
	24
	25	/*!
	26	* LEGACY_SUPPORT :
	27	* if set to 1+, ZSTD_decompress() can decode older formats (v0.1+)
	28	*/
	29	#ifndef ZSTD_LEGACY_SUPPORT
	30	# define ZSTD_LEGACY_SUPPORT 0
	31	#endif
	32
	33	/*!
	34	* MAXWINDOWSIZE_DEFAULT :
	35	* maximum window size accepted by DStream __by default__.
	36	* Frames requiring more memory will be rejected.
	37	* It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize().
	38	*/
	39	#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
	40	# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1)
	41	#endif
	42
	43	/*!
	44	* NO_FORWARD_PROGRESS_MAX :
	45	* maximum allowed nb of calls to ZSTD_decompressStream()
	46	* without any forward progress
	47	* (defined as: no byte read from input, and no byte flushed to output)
	48	* before triggering an error.
	49	*/
	50	#ifndef ZSTD_NO_FORWARD_PROGRESS_MAX
	51	# define ZSTD_NO_FORWARD_PROGRESS_MAX 16
	52	#endif
	53
	54
	55	/-******************************************************
	56	* Dependencies
	57	*********************************************************/
	58	#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
	59	#include "../common/mem.h" /* low level memory routines */
	60	#define FSE_STATIC_LINKING_ONLY
	61	#include "../common/fse.h"
	62	#define HUF_STATIC_LINKING_ONLY
	63	#include "../common/huf.h"
	64	#include "../common/xxhash.h" /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */
	65	#include "../common/zstd_internal.h" /* blockProperties_t */
	66	#include "zstd_decompress_internal.h" /* ZSTD_DCtx */
	67	#include "zstd_ddict.h" /* ZSTD_DDictDictContent */
	68	#include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */
	69
	70	#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
	71	# include "../legacy/zstd_legacy.h"
	72	#endif
	73
	74
	75
	76	/*************************************
	77	* Multiple DDicts Hashset internals *
	78	*************************************/
	79
	80	#define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4
	81	#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float.
	82	* Currently, that means a 0.75 load factor.
	83	* So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded
	84	* the load factor of the ddict hash set.
	85	*/
	86
	87	#define DDICT_HASHSET_TABLE_BASE_SIZE 64
	88	#define DDICT_HASHSET_RESIZE_FACTOR 2
	89
	90	/* Hash function to determine starting position of dict insertion within the table
	91	* Returns an index between [0, hashSet->ddictPtrTableSize]
	92	*/
	93	static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, U32 dictID) {
	94	const U64 hash = XXH64(&dictID, sizeof(U32), 0);
	95	/* DDict ptr table size is a multiple of 2, use size - 1 as mask to get index within [0, hashSet->ddictPtrTableSize) */
	96	return hash & (hashSet->ddictPtrTableSize - 1);
	97	}
	98
	99	/* Adds DDict to a hashset without resizing it.
	100	* If inserting a DDict with a dictID that already exists in the set, replaces the one in the set.
	101	* Returns 0 if successful, or a zstd error code if something went wrong.
	102	*/
	103	static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict) {
	104	const U32 dictID = ZSTD_getDictID_fromDDict(ddict);
	105	size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
	106	const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1;
	107	RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!");
	108	DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx);
	109	while (hashSet->ddictPtrTable[idx] != NULL) {
	110	/* Replace existing ddict if inserting ddict with same dictID */
	111	if (ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]) == dictID) {
	112	DEBUGLOG(4, "DictID already exists, replacing rather than adding");
	113	hashSet->ddictPtrTable[idx] = ddict;
	114	return 0;
	115	}
	116	idx &= idxRangeMask;
	117	idx++;
	118	}
	119	DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx);
	120	hashSet->ddictPtrTable[idx] = ddict;
	121	hashSet->ddictPtrCount++;
	122	return 0;
	123	}
	124
	125	/* Expands hash table by factor of DDICT_HASHSET_RESIZE_FACTOR and
	126	* rehashes all values, allocates new table, frees old table.
	127	* Returns 0 on success, otherwise a zstd error code.
	128	*/
	129	static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
	130	size_t newTableSize = hashSet->ddictPtrTableSize * DDICT_HASHSET_RESIZE_FACTOR;
	131	const ZSTD_DDict newTable = (const ZSTD_DDict)ZSTD_customCalloc(sizeof(ZSTD_DDict) newTableSize, customMem);
	132	const ZSTD_DDict** oldTable = hashSet->ddictPtrTable;
	133	size_t oldTableSize = hashSet->ddictPtrTableSize;
	134	size_t i;
	135
	136	DEBUGLOG(4, "Expanding DDict hash table! Old size: %zu new size: %zu", oldTableSize, newTableSize);
	137	RETURN_ERROR_IF(!newTable, memory_allocation, "Expanded hashset allocation failed!");
	138	hashSet->ddictPtrTable = newTable;
	139	hashSet->ddictPtrTableSize = newTableSize;
	140	hashSet->ddictPtrCount = 0;
	141	for (i = 0; i < oldTableSize; ++i) {
	142	if (oldTable[i] != NULL) {
	143	FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, oldTable[i]), "");
	144	}
	145	}
	146	ZSTD_customFree((void*)oldTable, customMem);
	147	DEBUGLOG(4, "Finished re-hash");
	148	return 0;
	149	}
	150
	151	/* Fetches a DDict with the given dictID
	152	* Returns the ZSTD_DDict* with the requested dictID. If it doesn't exist, then returns NULL.
	153	*/
	154	static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, U32 dictID) {
	155	size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID);
	156	const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1;
	157	DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx);
	158	for (;;) {
	159	size_t currDictID = ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]);
	160	if (currDictID == dictID \|\| currDictID == 0) {
	161	/* currDictID == 0 implies a NULL ddict entry */
	162	break;
	163	} else {
	164	idx &= idxRangeMask; /* Goes to start of table when we reach the end */
	165	idx++;
	166	}
	167	}
	168	DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx);
	169	return hashSet->ddictPtrTable[idx];
	170	}
	171
	172	/* Allocates space for and returns a ddict hash set
	173	* The hash set's ZSTD_DDict* table has all values automatically set to NULL to begin with.
	174	* Returns NULL if allocation failed.
	175	*/
	176	static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) {
	177	ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem);
	178	DEBUGLOG(4, "Allocating new hash set");
	179	if (!ret)
	180	return NULL;
	181	ret->ddictPtrTable = (const ZSTD_DDict*)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE sizeof(ZSTD_DDict*), customMem);
	182	if (!ret->ddictPtrTable) {
	183	ZSTD_customFree(ret, customMem);
	184	return NULL;
	185	}
	186	ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE;
	187	ret->ddictPtrCount = 0;
	188	return ret;
	189	}
	190
	191	/* Frees the table of ZSTD_DDict* within a hashset, then frees the hashset itself.
	192	* Note: The ZSTD_DDict* within the table are NOT freed.
	193	*/
	194	static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) {
	195	DEBUGLOG(4, "Freeing ddict hash set");
	196	if (hashSet && hashSet->ddictPtrTable) {
	197	ZSTD_customFree((void*)hashSet->ddictPtrTable, customMem);
	198	}
	199	if (hashSet) {
	200	ZSTD_customFree(hashSet, customMem);
	201	}
	202	}
	203
	204	/* Public function: Adds a DDict into the ZSTD_DDictHashSet, possibly triggering a resize of the hash set.
	205	* Returns 0 on success, or a ZSTD error.
	206	*/
	207	static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, ZSTD_customMem customMem) {
	208	DEBUGLOG(4, "Adding dict ID: %u to hashset with - Count: %zu Tablesize: %zu", ZSTD_getDictID_fromDDict(ddict), hashSet->ddictPtrCount, hashSet->ddictPtrTableSize);
	209	if (hashSet->ddictPtrCount * DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT / hashSet->ddictPtrTableSize * DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT != 0) {
	210	FORWARD_IF_ERROR(ZSTD_DDictHashSet_expand(hashSet, customMem), "");
	211	}
	212	FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, ddict), "");
	213	return 0;
	214	}
	215
	216	/-************************************************************
	217	* Context management
	218	***************************************************************/
	219	size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx)
	220	{
	221	if (dctx==NULL) return 0; /* support sizeof NULL */
	222	return sizeof(*dctx)
	223	+ ZSTD_sizeof_DDict(dctx->ddictLocal)
	224	+ dctx->inBuffSize + dctx->outBuffSize;
	225	}
	226
	227	size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
	228
	229
	230	static size_t ZSTD_startingInputLength(ZSTD_format_e format)
	231	{
	232	size_t const startingInputLength = ZSTD_FRAMEHEADERSIZE_PREFIX(format);
	233	/* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */
	234	assert( (format == ZSTD_f_zstd1) \|\| (format == ZSTD_f_zstd1_magicless) );
	235	return startingInputLength;
	236	}
	237
	238	static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx)
	239	{
	240	assert(dctx->streamStage == zdss_init);
	241	dctx->format = ZSTD_f_zstd1;
	242	dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
	243	dctx->outBufferMode = ZSTD_bm_buffered;
	244	dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum;
	245	dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict;
	246	}
	247
	248	static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
	249	{
	250	dctx->staticSize = 0;
	251	dctx->ddict = NULL;
	252	dctx->ddictLocal = NULL;
	253	dctx->dictEnd = NULL;
	254	dctx->ddictIsCold = 0;
	255	dctx->dictUses = ZSTD_dont_use;
	256	dctx->inBuff = NULL;
	257	dctx->inBuffSize = 0;
	258	dctx->outBuffSize = 0;
	259	dctx->streamStage = zdss_init;
	260	#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
	261	dctx->legacyContext = NULL;
	262	dctx->previousLegacyVersion = 0;
	263	#endif
	264	dctx->noForwardProgress = 0;
	265	dctx->oversizedDuration = 0;
	266	#if DYNAMIC_BMI2
	267	dctx->bmi2 = ZSTD_cpuSupportsBmi2();
	268	#endif
	269	dctx->ddictSet = NULL;
	270	ZSTD_DCtx_resetParameters(dctx);
	271	#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
	272	dctx->dictContentEndForFuzzing = NULL;
	273	#endif
	274	}
	275
	276	ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize)
	277	{
	278	ZSTD_DCtx* const dctx = (ZSTD_DCtx*) workspace;
	279
	280	if ((size_t)workspace & 7) return NULL; /* 8-aligned */
	281	if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL; /* minimum size */
	282
	283	ZSTD_initDCtx_internal(dctx);
	284	dctx->staticSize = workspaceSize;
	285	dctx->inBuff = (char*)(dctx+1);
	286	return dctx;
	287	}
	288
	289	static ZSTD_DCtx* ZSTD_createDCtx_internal(ZSTD_customMem customMem) {
	290	if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
	291
	292	{ ZSTD_DCtx* const dctx = (ZSTD_DCtx)ZSTD_customMalloc(sizeof(dctx), customMem);
	293	if (!dctx) return NULL;
	294	dctx->customMem = customMem;
	295	ZSTD_initDCtx_internal(dctx);
	296	return dctx;
	297	}
	298	}
	299
	300	ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
	301	{
	302	return ZSTD_createDCtx_internal(customMem);
	303	}
	304
	305	ZSTD_DCtx* ZSTD_createDCtx(void)
	306	{
	307	DEBUGLOG(3, "ZSTD_createDCtx");
	308	return ZSTD_createDCtx_internal(ZSTD_defaultCMem);
	309	}
	310
	311	static void ZSTD_clearDict(ZSTD_DCtx* dctx)
	312	{
	313	ZSTD_freeDDict(dctx->ddictLocal);
	314	dctx->ddictLocal = NULL;
	315	dctx->ddict = NULL;
	316	dctx->dictUses = ZSTD_dont_use;
	317	}
	318
	319	size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
	320	{
	321	if (dctx==NULL) return 0; /* support free on NULL */
	322	RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx");
	323	{ ZSTD_customMem const cMem = dctx->customMem;
	324	ZSTD_clearDict(dctx);
	325	ZSTD_customFree(dctx->inBuff, cMem);
	326	dctx->inBuff = NULL;
	327	#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
	328	if (dctx->legacyContext)
	329	ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion);
	330	#endif
	331	if (dctx->ddictSet) {
	332	ZSTD_freeDDictHashSet(dctx->ddictSet, cMem);
	333	dctx->ddictSet = NULL;
	334	}
	335	ZSTD_customFree(dctx, cMem);
	336	return 0;
	337	}
	338	}
	339
	340	/* no longer useful */
	341	void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
	342	{
	343	size_t const toCopy = (size_t)((char)(&dstDCtx->inBuff) - (char)dstDCtx);
	344	ZSTD_memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */
	345	}
	346
	347	/* Given a dctx with a digested frame params, re-selects the correct ZSTD_DDict based on
	348	* the requested dict ID from the frame. If there exists a reference to the correct ZSTD_DDict, then
	349	* accordingly sets the ddict to be used to decompress the frame.
	350	*
	351	* If no DDict is found, then no action is taken, and the ZSTD_DCtx::ddict remains as-is.
	352	*
	353	* ZSTD_d_refMultipleDDicts must be enabled for this function to be called.
	354	*/
	355	static void ZSTD_DCtx_selectFrameDDict(ZSTD_DCtx* dctx) {
	356	assert(dctx->refMultipleDDicts && dctx->ddictSet);
	357	DEBUGLOG(4, "Adjusting DDict based on requested dict ID from frame");
	358	if (dctx->ddict) {
	359	const ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(dctx->ddictSet, dctx->fParams.dictID);
	360	if (frameDDict) {
	361	DEBUGLOG(4, "DDict found!");
	362	ZSTD_clearDict(dctx);
	363	dctx->dictID = dctx->fParams.dictID;
	364	dctx->ddict = frameDDict;
	365	dctx->dictUses = ZSTD_use_indefinitely;
	366	}
	367	}
	368	}
	369
	370
	371	/-************************************************************
	372	* Frame header decoding
	373	***************************************************************/
	374
	375	/*! ZSTD_isFrame() :
	376	* Tells if the content of `buffer` starts with a valid Frame Identifier.
	377	* Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
	378	* Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
	379	* Note 3 : Skippable Frame Identifiers are considered valid. */
	380	unsigned ZSTD_isFrame(const void* buffer, size_t size)
	381	{
	382	if (size < ZSTD_FRAMEIDSIZE) return 0;
	383	{ U32 const magic = MEM_readLE32(buffer);
	384	if (magic == ZSTD_MAGICNUMBER) return 1;
	385	if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
	386	}
	387	#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
	388	if (ZSTD_isLegacy(buffer, size)) return 1;
	389	#endif
	390	return 0;
	391	}
	392
	393	/*! ZSTD_isSkippableFrame() :
	394	* Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
	395	* Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
	396	*/
	397	unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size)
	398	{
	399	if (size < ZSTD_FRAMEIDSIZE) return 0;
	400	{ U32 const magic = MEM_readLE32(buffer);
	401	if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
	402	}
	403	return 0;
	404	}
	405
	406	/** ZSTD_frameHeaderSize_internal() :
	407	* srcSize must be large enough to reach header size fields.
	408	* note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless.
	409	* @return : size of the Frame Header
	410	* or an error code, which can be tested with ZSTD_isError() */
	411	static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format)
	412	{
	413	size_t const minInputSize = ZSTD_startingInputLength(format);
	414	RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong, "");
	415
	416	{ BYTE const fhd = ((const BYTE*)src)[minInputSize-1];
	417	U32 const dictID= fhd & 3;
	418	U32 const singleSegment = (fhd >> 5) & 1;
	419	U32 const fcsId = fhd >> 6;
	420	return minInputSize + !singleSegment
	421	+ ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId]
	422	+ (singleSegment && !fcsId);
	423	}
	424	}
	425
	426	/** ZSTD_frameHeaderSize() :
	427	* srcSize must be >= ZSTD_frameHeaderSize_prefix.
	428	* @return : size of the Frame Header,
	429	* or an error code (if srcSize is too small) */
	430	size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
	431	{
	432	return ZSTD_frameHeaderSize_internal(src, srcSize, ZSTD_f_zstd1);
	433	}
	434
	435
	436	/** ZSTD_getFrameHeader_advanced() :
	437	* decode Frame Header, or require larger `srcSize`.
	438	* note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless
	439	* @return : 0, `zfhPtr` is correctly filled,
	440	* >0, `srcSize` is too small, value is wanted `srcSize` amount,
	441	* or an error code, which can be tested using ZSTD_isError() */
	442	size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format)
	443	{
	444	const BYTE* ip = (const BYTE*)src;
	445	size_t const minInputSize = ZSTD_startingInputLength(format);
	446
	447	ZSTD_memset(zfhPtr, 0, sizeof(zfhPtr)); / not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */
	448	if (srcSize < minInputSize) return minInputSize;
	449	RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter");
	450
	451	if ( (format != ZSTD_f_zstd1_magicless)
	452	&& (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) {
	453	if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
	454	/* skippable frame */
	455	if (srcSize < ZSTD_SKIPPABLEHEADERSIZE)
	456	return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */
	457	ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr));
	458	zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE);
	459	zfhPtr->frameType = ZSTD_skippableFrame;
	460	return 0;
	461	}
	462	RETURN_ERROR(prefix_unknown, "");
	463	}
	464
	465	/* ensure there is enough `srcSize` to fully read/decode frame header */
	466	{ size_t const fhsize = ZSTD_frameHeaderSize_internal(src, srcSize, format);
	467	if (srcSize < fhsize) return fhsize;
	468	zfhPtr->headerSize = (U32)fhsize;
	469	}
	470
	471	{ BYTE const fhdByte = ip[minInputSize-1];
	472	size_t pos = minInputSize;
	473	U32 const dictIDSizeCode = fhdByte&3;
	474	U32 const checksumFlag = (fhdByte>>2)&1;
	475	U32 const singleSegment = (fhdByte>>5)&1;
	476	U32 const fcsID = fhdByte>>6;
	477	U64 windowSize = 0;
	478	U32 dictID = 0;
	479	U64 frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN;
	480	RETURN_ERROR_IF((fhdByte & 0x08) != 0, frameParameter_unsupported,
	481	"reserved bits, must be zero");
	482
	483	if (!singleSegment) {
	484	BYTE const wlByte = ip[pos++];
	485	U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
	486	RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge, "");
	487	windowSize = (1ULL << windowLog);
	488	windowSize += (windowSize >> 3) * (wlByte&7);
	489	}
	490	switch(dictIDSizeCode)
	491	{
	492	default:
	493	assert(0); /* impossible */
	494	ZSTD_FALLTHROUGH;
	495	case 0 : break;
	496	case 1 : dictID = ip[pos]; pos++; break;
	497	case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break;
	498	case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break;
	499	}
	500	switch(fcsID)
	501	{
	502	default:
	503	assert(0); /* impossible */
	504	ZSTD_FALLTHROUGH;
	505	case 0 : if (singleSegment) frameContentSize = ip[pos]; break;
	506	case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break;
	507	case 2 : frameContentSize = MEM_readLE32(ip+pos); break;
	508	case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
	509	}
	510	if (singleSegment) windowSize = frameContentSize;
	511
	512	zfhPtr->frameType = ZSTD_frame;
	513	zfhPtr->frameContentSize = frameContentSize;
	514	zfhPtr->windowSize = windowSize;
	515	zfhPtr->blockSizeMax = (unsigned) MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
	516	zfhPtr->dictID = dictID;
	517	zfhPtr->checksumFlag = checksumFlag;
	518	}
	519	return 0;
	520	}
	521
	522	/** ZSTD_getFrameHeader() :
	523	* decode Frame Header, or require larger `srcSize`.
	524	* note : this function does not consume input, it only reads it.
	525	* @return : 0, `zfhPtr` is correctly filled,
	526	* >0, `srcSize` is too small, value is wanted `srcSize` amount,
	527	* or an error code, which can be tested using ZSTD_isError() */
	528	size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize)
	529	{
	530	return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1);
	531	}
	532
	533	/** ZSTD_getFrameContentSize() :
	534	* compatible with legacy mode
	535	* @return : decompressed size of the single frame pointed to be `src` if known, otherwise
	536	* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
	537	* - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */
	538	unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
	539	{
	540	#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
	541	if (ZSTD_isLegacy(src, srcSize)) {
	542	unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize);
	543	return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret;
	544	}
	545	#endif
	546	{ ZSTD_frameHeader zfh;
	547	if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0)
	548	return ZSTD_CONTENTSIZE_ERROR;
	549	if (zfh.frameType == ZSTD_skippableFrame) {
	550	return 0;
	551	} else {
	552	return zfh.frameContentSize;
	553	} }
	554	}
	555
	556	static size_t readSkippableFrameSize(void const* src, size_t srcSize)
	557	{
	558	size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE;
	559	U32 sizeU32;
	560
	561	RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, "");
	562
	563	sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE);
	564	RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32,
	565	frameParameter_unsupported, "");
	566	{
	567	size_t const skippableSize = skippableHeaderSize + sizeU32;
	568	RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, "");
	569	return skippableSize;
	570	}
	571	}
	572
	573	/*! ZSTD_readSkippableFrame() :
	574	* Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
	575	*
	576	* The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
	577	* i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested
	578	* in the magicVariant.
	579	*
	580	* Returns an error if destination buffer is not large enough, or if the frame is not skippable.
	581	*
	582	* @return : number of bytes written or a ZSTD error.
	583	*/
	584	ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
	585	const void* src, size_t srcSize)
	586	{
	587	U32 const magicNumber = MEM_readLE32(src);
	588	size_t skippableFrameSize = readSkippableFrameSize(src, srcSize);
	589	size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE;
	590
	591	/* check input validity */
	592	RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, "");
	593	RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE \|\| skippableFrameSize > srcSize, srcSize_wrong, "");
	594	RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, "");
	595
	596	/* deliver payload */
	597	if (skippableContentSize > 0 && dst != NULL)
	598	ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize);
	599	if (magicVariant != NULL)
	600	*magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START;
	601	return skippableContentSize;
	602	}
	603
	604	/** ZSTD_findDecompressedSize() :
	605	* compatible with legacy mode
	606	* `srcSize` must be the exact length of some number of ZSTD compressed and/or
	607	* skippable frames
	608	* @return : decompressed size of the frames contained */
	609	unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
	610	{
	611	unsigned long long totalDstSize = 0;
	612
	613	while (srcSize >= ZSTD_startingInputLength(ZSTD_f_zstd1)) {
	614	U32 const magicNumber = MEM_readLE32(src);
	615
	616	if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
	617	size_t const skippableSize = readSkippableFrameSize(src, srcSize);
	618	if (ZSTD_isError(skippableSize)) {
	619	return ZSTD_CONTENTSIZE_ERROR;
	620	}
	621	assert(skippableSize <= srcSize);
	622
	623	src = (const BYTE *)src + skippableSize;
	624	srcSize -= skippableSize;
	625	continue;
	626	}
	627
	628	{ unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
	629	if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret;
	630
	631	/* check for overflow */
	632	if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR;
	633	totalDstSize += ret;
	634	}
	635	{ size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize);
	636	if (ZSTD_isError(frameSrcSize)) {
	637	return ZSTD_CONTENTSIZE_ERROR;
	638	}
	639
	640	src = (const BYTE *)src + frameSrcSize;
	641	srcSize -= frameSrcSize;
	642	}
	643	} /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */
	644
	645	if (srcSize) return ZSTD_CONTENTSIZE_ERROR;
	646
	647	return totalDstSize;
	648	}
	649
	650	/** ZSTD_getDecompressedSize() :
	651	* compatible with legacy mode
	652	* @return : decompressed size if known, 0 otherwise
	653	note : 0 can mean any of the following :
	654	- frame content is empty
	655	- decompressed size field is not present in frame header
	656	- frame header unknown / not supported
	657	- frame header not complete (`srcSize` too small) */
	658	unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
	659	{
	660	unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
	661	ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_ERROR < ZSTD_CONTENTSIZE_UNKNOWN);
	662	return (ret >= ZSTD_CONTENTSIZE_ERROR) ? 0 : ret;
	663	}
	664
	665
	666	/** ZSTD_decodeFrameHeader() :
	667	* `headerSize` must be the size provided by ZSTD_frameHeaderSize().
	668	* If multiple DDict references are enabled, also will choose the correct DDict to use.
	669	* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
	670	static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize)
	671	{
	672	size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format);
	673	if (ZSTD_isError(result)) return result; /* invalid header */
	674	RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small");
	675
	676	/* Reference DDict requested by frame if dctx references multiple ddicts */
	677	if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts && dctx->ddictSet) {
	678	ZSTD_DCtx_selectFrameDDict(dctx);
	679	}
	680
	681	#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
	682	/* Skip the dictID check in fuzzing mode, because it makes the search
	683	* harder.
	684	*/
	685	RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID),
	686	dictionary_wrong, "");
	687	#endif
	688	dctx->validateChecksum = (dctx->fParams.checksumFlag && !dctx->forceIgnoreChecksum) ? 1 : 0;
	689	if (dctx->validateChecksum) XXH64_reset(&dctx->xxhState, 0);
	690	dctx->processedCSize += headerSize;
	691	return 0;
	692	}
	693
	694	static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret)
	695	{
	696	ZSTD_frameSizeInfo frameSizeInfo;
	697	frameSizeInfo.compressedSize = ret;
	698	frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
	699	return frameSizeInfo;
	700	}
	701
	702	static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize)
	703	{
	704	ZSTD_frameSizeInfo frameSizeInfo;
	705	ZSTD_memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo));
	706
	707	#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
	708	if (ZSTD_isLegacy(src, srcSize))
	709	return ZSTD_findFrameSizeInfoLegacy(src, srcSize);
	710	#endif
	711
	712	if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE)
	713	&& (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
	714	frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize);
	715	assert(ZSTD_isError(frameSizeInfo.compressedSize) \|\|
	716	frameSizeInfo.compressedSize <= srcSize);
	717	return frameSizeInfo;
	718	} else {
	719	const BYTE* ip = (const BYTE*)src;
	720	const BYTE* const ipstart = ip;
	721	size_t remainingSize = srcSize;
	722	size_t nbBlocks = 0;
	723	ZSTD_frameHeader zfh;
	724
	725	/* Extract Frame Header */
	726	{ size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize);
	727	if (ZSTD_isError(ret))
	728	return ZSTD_errorFrameSizeInfo(ret);
	729	if (ret > 0)
	730	return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong));
	731	}
	732
	733	ip += zfh.headerSize;
	734	remainingSize -= zfh.headerSize;
	735
	736	/* Iterate over each block */
	737	while (1) {
	738	blockProperties_t blockProperties;
	739	size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
	740	if (ZSTD_isError(cBlockSize))
	741	return ZSTD_errorFrameSizeInfo(cBlockSize);
	742
	743	if (ZSTD_blockHeaderSize + cBlockSize > remainingSize)
	744	return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong));
	745
	746	ip += ZSTD_blockHeaderSize + cBlockSize;
	747	remainingSize -= ZSTD_blockHeaderSize + cBlockSize;
	748	nbBlocks++;
	749
	750	if (blockProperties.lastBlock) break;
	751	}
	752
	753	/* Final frame content checksum */
	754	if (zfh.checksumFlag) {
	755	if (remainingSize < 4)
	756	return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong));
	757	ip += 4;
	758	}
	759
	760	frameSizeInfo.compressedSize = (size_t)(ip - ipstart);
	761	frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN)
	762	? zfh.frameContentSize
	763	: nbBlocks * zfh.blockSizeMax;
	764	return frameSizeInfo;
	765	}
	766	}
	767
	768	/** ZSTD_findFrameCompressedSize() :
	769	* compatible with legacy mode
	770	* `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame
	771	* `srcSize` must be at least as large as the frame contained
	772	* @return : the compressed size of the frame starting at `src` */
	773	size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
	774	{
	775	ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
	776	return frameSizeInfo.compressedSize;
	777	}
	778
	779	/** ZSTD_decompressBound() :
	780	* compatible with legacy mode
	781	* `src` must point to the start of a ZSTD frame or a skippeable frame
	782	* `srcSize` must be at least as large as the frame contained
	783	* @return : the maximum decompressed size of the compressed source
	784	*/
	785	unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
	786	{
	787	unsigned long long bound = 0;
	788	/* Iterate over each frame */
	789	while (srcSize > 0) {
	790	ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
	791	size_t const compressedSize = frameSizeInfo.compressedSize;
	792	unsigned long long const decompressedBound = frameSizeInfo.decompressedBound;
	793	if (ZSTD_isError(compressedSize) \|\| decompressedBound == ZSTD_CONTENTSIZE_ERROR)
	794	return ZSTD_CONTENTSIZE_ERROR;
	795	assert(srcSize >= compressedSize);
	796	src = (const BYTE*)src + compressedSize;
	797	srcSize -= compressedSize;
	798	bound += decompressedBound;
	799	}
	800	return bound;
	801	}
	802
	803
	804	/-************************************************************
	805	* Frame decoding
	806	***************************************************************/
	807
	808	/** ZSTD_insertBlock() :
	809	* insert `src` block into `dctx` history. Useful to track uncompressed blocks. */
	810	size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize)
	811	{
	812	DEBUGLOG(5, "ZSTD_insertBlock: %u bytes", (unsigned)blockSize);
	813	ZSTD_checkContinuity(dctx, blockStart, blockSize);
	814	dctx->previousDstEnd = (const char*)blockStart + blockSize;
	815	return blockSize;
	816	}
	817
	818
	819	static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity,
	820	const void* src, size_t srcSize)
	821	{
	822	DEBUGLOG(5, "ZSTD_copyRawBlock");
	823	RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, "");
	824	if (dst == NULL) {
	825	if (srcSize == 0) return 0;
	826	RETURN_ERROR(dstBuffer_null, "");
	827	}
	828	ZSTD_memcpy(dst, src, srcSize);
	829	return srcSize;
	830	}
	831
	832	static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity,
	833	BYTE b,
	834	size_t regenSize)
	835	{
	836	RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, "");
	837	if (dst == NULL) {
	838	if (regenSize == 0) return 0;
	839	RETURN_ERROR(dstBuffer_null, "");
	840	}
	841	ZSTD_memset(dst, b, regenSize);
	842	return regenSize;
	843	}
	844
	845	static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming)
	846	{
	847	#if ZSTD_TRACE
	848	if (dctx->traceCtx && ZSTD_trace_decompress_end != NULL) {
	849	ZSTD_Trace trace;
	850	ZSTD_memset(&trace, 0, sizeof(trace));
	851	trace.version = ZSTD_VERSION_NUMBER;
	852	trace.streaming = streaming;
	853	if (dctx->ddict) {
	854	trace.dictionaryID = ZSTD_getDictID_fromDDict(dctx->ddict);
	855	trace.dictionarySize = ZSTD_DDict_dictSize(dctx->ddict);
	856	trace.dictionaryIsCold = dctx->ddictIsCold;
	857	}
	858	trace.uncompressedSize = (size_t)uncompressedSize;
	859	trace.compressedSize = (size_t)compressedSize;
	860	trace.dctx = dctx;
	861	ZSTD_trace_decompress_end(dctx->traceCtx, &trace);
	862	}
	863	#else
	864	(void)dctx;
	865	(void)uncompressedSize;
	866	(void)compressedSize;
	867	(void)streaming;
	868	#endif
	869	}
	870
	871
	872	/*! ZSTD_decompressFrame() :
	873	* @dctx must be properly initialized
	874	* will update srcPtr and srcSizePtr,
	875	* to make srcPtr progress by one frame. /
	876	static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
	877	void* dst, size_t dstCapacity,
	878	const void** srcPtr, size_t *srcSizePtr)
	879	{
	880	const BYTE* const istart = (const BYTE)(srcPtr);
	881	const BYTE* ip = istart;
	882	BYTE* const ostart = (BYTE*)dst;
	883	BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart;
	884	BYTE* op = ostart;
	885	size_t remainingSrcSize = *srcSizePtr;
	886
	887	DEBUGLOG(4, "ZSTD_decompressFrame (srcSize:%i)", (int)*srcSizePtr);
	888
	889	/* check */
	890	RETURN_ERROR_IF(
	891	remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN(dctx->format)+ZSTD_blockHeaderSize,
	892	srcSize_wrong, "");
	893
	894	/* Frame Header */
	895	{ size_t const frameHeaderSize = ZSTD_frameHeaderSize_internal(
	896	ip, ZSTD_FRAMEHEADERSIZE_PREFIX(dctx->format), dctx->format);
	897	if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
	898	RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize,
	899	srcSize_wrong, "");
	900	FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) , "");
	901	ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize;
	902	}
	903
	904	/* Loop on each block */
	905	while (1) {
	906	size_t decodedSize;
	907	blockProperties_t blockProperties;
	908	size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties);
	909	if (ZSTD_isError(cBlockSize)) return cBlockSize;
	910
	911	ip += ZSTD_blockHeaderSize;
	912	remainingSrcSize -= ZSTD_blockHeaderSize;
	913	RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, "");
	914
	915	switch(blockProperties.blockType)
	916	{
	917	case bt_compressed:
	918	decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oend-op), ip, cBlockSize, /* frame */ 1, not_streaming);
	919	break;
	920	case bt_raw :
	921	decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize);
	922	break;
	923	case bt_rle :
	924	decodedSize = ZSTD_setRleBlock(op, (size_t)(oend-op), *ip, blockProperties.origSize);
	925	break;
	926	case bt_reserved :
	927	default:
	928	RETURN_ERROR(corruption_detected, "invalid block type");
	929	}
	930
	931	if (ZSTD_isError(decodedSize)) return decodedSize;
	932	if (dctx->validateChecksum)
	933	XXH64_update(&dctx->xxhState, op, decodedSize);
	934	if (decodedSize != 0)
	935	op += decodedSize;
	936	assert(ip != NULL);
	937	ip += cBlockSize;
	938	remainingSrcSize -= cBlockSize;
	939	if (blockProperties.lastBlock) break;
	940	}
	941
	942	if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) {
	943	RETURN_ERROR_IF((U64)(op-ostart) != dctx->fParams.frameContentSize,
	944	corruption_detected, "");
	945	}
	946	if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */
	947	RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong, "");
	948	if (!dctx->forceIgnoreChecksum) {
	949	U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState);
	950	U32 checkRead;
	951	checkRead = MEM_readLE32(ip);
	952	RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, "");
	953	}
	954	ip += 4;
	955	remainingSrcSize -= 4;
	956	}
	957	ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0);
	958	/* Allow caller to get size read */
	959	*srcPtr = ip;
	960	*srcSizePtr = remainingSrcSize;
	961	return (size_t)(op-ostart);
	962	}
	963
	964	static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
	965	void* dst, size_t dstCapacity,
	966	const void* src, size_t srcSize,
	967	const void* dict, size_t dictSize,
	968	const ZSTD_DDict* ddict)
	969	{
	970	void* const dststart = dst;
	971	int moreThan1Frame = 0;
	972
	973	DEBUGLOG(5, "ZSTD_decompressMultiFrame");
	974	assert(dict==NULL \|\| ddict==NULL); /* either dict or ddict set, not both */
	975
	976	if (ddict) {
	977	dict = ZSTD_DDict_dictContent(ddict);
	978	dictSize = ZSTD_DDict_dictSize(ddict);
	979	}
	980
	981	while (srcSize >= ZSTD_startingInputLength(dctx->format)) {
	982
	983	#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
	984	if (ZSTD_isLegacy(src, srcSize)) {
	985	size_t decodedSize;
	986	size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
	987	if (ZSTD_isError(frameSize)) return frameSize;
	988	RETURN_ERROR_IF(dctx->staticSize, memory_allocation,
	989	"legacy support is not compatible with static dctx");
	990
	991	decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize);
	992	if (ZSTD_isError(decodedSize)) return decodedSize;
	993
	994	assert(decodedSize <= dstCapacity);
	995	dst = (BYTE*)dst + decodedSize;
	996	dstCapacity -= decodedSize;
	997
	998	src = (const BYTE*)src + frameSize;
	999	srcSize -= frameSize;
	1000
	1001	continue;
	1002	}
	1003	#endif
	1004
	1005	{ U32 const magicNumber = MEM_readLE32(src);
	1006	DEBUGLOG(4, "reading magic number %08X (expecting %08X)",
	1007	(unsigned)magicNumber, ZSTD_MAGICNUMBER);
	1008	if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
	1009	size_t const skippableSize = readSkippableFrameSize(src, srcSize);
	1010	FORWARD_IF_ERROR(skippableSize, "readSkippableFrameSize failed");
	1011	assert(skippableSize <= srcSize);
	1012
	1013	src = (const BYTE *)src + skippableSize;
	1014	srcSize -= skippableSize;
	1015	continue;
	1016	} }
	1017
	1018	if (ddict) {
	1019	/* we were called from ZSTD_decompress_usingDDict */
	1020	FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict), "");
	1021	} else {
	1022	/* this will initialize correctly with no dict if dict == NULL, so
	1023	* use this in all cases but ddict */
	1024	FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize), "");
	1025	}
	1026	ZSTD_checkContinuity(dctx, dst, dstCapacity);
	1027
	1028	{ const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity,
	1029	&src, &srcSize);
	1030	RETURN_ERROR_IF(
	1031	(ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown)
	1032	&& (moreThan1Frame==1),
	1033	srcSize_wrong,
	1034	"At least one frame successfully completed, "
	1035	"but following bytes are garbage: "
	1036	"it's more likely to be a srcSize error, "
	1037	"specifying more input bytes than size of frame(s). "
	1038	"Note: one could be unlucky, it might be a corruption error instead, "
	1039	"happening right at the place where we expect zstd magic bytes. "
	1040	"But this is _much_ less likely than a srcSize field error.");
	1041	if (ZSTD_isError(res)) return res;
	1042	assert(res <= dstCapacity);
	1043	if (res != 0)
	1044	dst = (BYTE*)dst + res;
	1045	dstCapacity -= res;
	1046	}
	1047	moreThan1Frame = 1;
	1048	} /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */
	1049
	1050	RETURN_ERROR_IF(srcSize, srcSize_wrong, "input not entirely consumed");
	1051
	1052	return (size_t)((BYTE)dst - (BYTE)dststart);
	1053	}
	1054
	1055	size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
	1056	void* dst, size_t dstCapacity,
	1057	const void* src, size_t srcSize,
	1058	const void* dict, size_t dictSize)
	1059	{
	1060	return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL);
	1061	}
	1062
	1063
	1064	static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx)
	1065	{
	1066	switch (dctx->dictUses) {
	1067	default:
	1068	assert(0 /* Impossible */);
	1069	ZSTD_FALLTHROUGH;
	1070	case ZSTD_dont_use:
	1071	ZSTD_clearDict(dctx);
	1072	return NULL;
	1073	case ZSTD_use_indefinitely:
	1074	return dctx->ddict;
	1075	case ZSTD_use_once:
	1076	dctx->dictUses = ZSTD_dont_use;
	1077	return dctx->ddict;
	1078	}
	1079	}
	1080
	1081	size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
	1082	{
	1083	return ZSTD_decompress_usingDDict(dctx, dst, dstCapacity, src, srcSize, ZSTD_getDDict(dctx));
	1084	}
	1085
	1086
	1087	size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
	1088	{
	1089	#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1)
	1090	size_t regenSize;
	1091	ZSTD_DCtx* const dctx = ZSTD_createDCtx_internal(ZSTD_defaultCMem);
	1092	RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!");
	1093	regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize);
	1094	ZSTD_freeDCtx(dctx);
	1095	return regenSize;
	1096	#else /* stack mode */
	1097	ZSTD_DCtx dctx;
	1098	ZSTD_initDCtx_internal(&dctx);
	1099	return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize);
	1100	#endif
	1101	}
	1102
	1103
	1104	/-*************************************
	1105	* Advanced Streaming Decompression API
	1106	* Bufferless and synchronous
	1107	****************************************/
	1108	size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; }
	1109
	1110	/**
	1111	* Similar to ZSTD_nextSrcSizeToDecompress(), but when when a block input can be streamed,
	1112	* we allow taking a partial block as the input. Currently only raw uncompressed blocks can
	1113	* be streamed.
	1114	*
	1115	* For blocks that can be streamed, this allows us to reduce the latency until we produce
	1116	* output, and avoid copying the input.
	1117	*
	1118	* @param inputSize - The total amount of input that the caller currently has.
	1119	*/
	1120	static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t inputSize) {
	1121	if (!(dctx->stage == ZSTDds_decompressBlock \|\| dctx->stage == ZSTDds_decompressLastBlock))
	1122	return dctx->expected;
	1123	if (dctx->bType != bt_raw)
	1124	return dctx->expected;
	1125	return BOUNDED(1, inputSize, dctx->expected);
	1126	}
	1127
	1128	ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) {
	1129	switch(dctx->stage)
	1130	{
	1131	default: /* should not happen */
	1132	assert(0);
	1133	ZSTD_FALLTHROUGH;
	1134	case ZSTDds_getFrameHeaderSize:
	1135	ZSTD_FALLTHROUGH;
	1136	case ZSTDds_decodeFrameHeader:
	1137	return ZSTDnit_frameHeader;
	1138	case ZSTDds_decodeBlockHeader:
	1139	return ZSTDnit_blockHeader;
	1140	case ZSTDds_decompressBlock:
	1141	return ZSTDnit_block;
	1142	case ZSTDds_decompressLastBlock:
	1143	return ZSTDnit_lastBlock;
	1144	case ZSTDds_checkChecksum:
	1145	return ZSTDnit_checksum;
	1146	case ZSTDds_decodeSkippableHeader:
	1147	ZSTD_FALLTHROUGH;
	1148	case ZSTDds_skipFrame:
	1149	return ZSTDnit_skippableFrame;
	1150	}
	1151	}
	1152
	1153	static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; }
	1154
	1155	/** ZSTD_decompressContinue() :
	1156	* srcSize : must be the exact nb of bytes expected (see ZSTD_nextSrcSizeToDecompress())
	1157	* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity)
	1158	* or an error code, which can be tested using ZSTD_isError() */
	1159	size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
	1160	{
	1161	DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize);
	1162	/* Sanity check */
	1163	RETURN_ERROR_IF(srcSize != ZSTD_nextSrcSizeToDecompressWithInputSize(dctx, srcSize), srcSize_wrong, "not allowed");
	1164	ZSTD_checkContinuity(dctx, dst, dstCapacity);
	1165
	1166	dctx->processedCSize += srcSize;
	1167
	1168	switch (dctx->stage)
	1169	{
	1170	case ZSTDds_getFrameHeaderSize :
	1171	assert(src != NULL);
	1172	if (dctx->format == ZSTD_f_zstd1) { /* allows header */
	1173	assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */
	1174	if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
	1175	ZSTD_memcpy(dctx->headerBuffer, src, srcSize);
	1176	dctx->expected = ZSTD_SKIPPABLEHEADERSIZE - srcSize; /* remaining to load to get full skippable frame header */
	1177	dctx->stage = ZSTDds_decodeSkippableHeader;
	1178	return 0;
	1179	} }
	1180	dctx->headerSize = ZSTD_frameHeaderSize_internal(src, srcSize, dctx->format);
	1181	if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
	1182	ZSTD_memcpy(dctx->headerBuffer, src, srcSize);
	1183	dctx->expected = dctx->headerSize - srcSize;
	1184	dctx->stage = ZSTDds_decodeFrameHeader;
	1185	return 0;
	1186
	1187	case ZSTDds_decodeFrameHeader:
	1188	assert(src != NULL);
	1189	ZSTD_memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize);
	1190	FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize), "");
	1191	dctx->expected = ZSTD_blockHeaderSize;
	1192	dctx->stage = ZSTDds_decodeBlockHeader;
	1193	return 0;
	1194
	1195	case ZSTDds_decodeBlockHeader:
	1196	{ blockProperties_t bp;
	1197	size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
	1198	if (ZSTD_isError(cBlockSize)) return cBlockSize;
	1199	RETURN_ERROR_IF(cBlockSize > dctx->fParams.blockSizeMax, corruption_detected, "Block Size Exceeds Maximum");
	1200	dctx->expected = cBlockSize;
	1201	dctx->bType = bp.blockType;
	1202	dctx->rleSize = bp.origSize;
	1203	if (cBlockSize) {
	1204	dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock;
	1205	return 0;
	1206	}
	1207	/* empty block */
	1208	if (bp.lastBlock) {
	1209	if (dctx->fParams.checksumFlag) {
	1210	dctx->expected = 4;
	1211	dctx->stage = ZSTDds_checkChecksum;
	1212	} else {
	1213	dctx->expected = 0; /* end of frame */
	1214	dctx->stage = ZSTDds_getFrameHeaderSize;
	1215	}
	1216	} else {
	1217	dctx->expected = ZSTD_blockHeaderSize; /* jump to next header */
	1218	dctx->stage = ZSTDds_decodeBlockHeader;
	1219	}
	1220	return 0;
	1221	}
	1222
	1223	case ZSTDds_decompressLastBlock:
	1224	case ZSTDds_decompressBlock:
	1225	DEBUGLOG(5, "ZSTD_decompressContinue: case ZSTDds_decompressBlock");
	1226	{ size_t rSize;
	1227	switch(dctx->bType)
	1228	{
	1229	case bt_compressed:
	1230	DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed");
	1231	rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1, is_streaming);
	1232	dctx->expected = 0; /* Streaming not supported */
	1233	break;
	1234	case bt_raw :
	1235	assert(srcSize <= dctx->expected);
	1236	rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize);
	1237	FORWARD_IF_ERROR(rSize, "ZSTD_copyRawBlock failed");
	1238	assert(rSize == srcSize);
	1239	dctx->expected -= rSize;
	1240	break;
	1241	case bt_rle :
	1242	rSize = ZSTD_setRleBlock(dst, dstCapacity, (const BYTE)src, dctx->rleSize);
	1243	dctx->expected = 0; /* Streaming not supported */
	1244	break;
	1245	case bt_reserved : /* should never happen */
	1246	default:
	1247	RETURN_ERROR(corruption_detected, "invalid block type");
	1248	}
	1249	FORWARD_IF_ERROR(rSize, "");
	1250	RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum");
	1251	DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize);
	1252	dctx->decodedSize += rSize;
	1253	if (dctx->validateChecksum) XXH64_update(&dctx->xxhState, dst, rSize);
	1254	dctx->previousDstEnd = (char*)dst + rSize;
	1255
	1256	/* Stay on the same stage until we are finished streaming the block. */
	1257	if (dctx->expected > 0) {
	1258	return rSize;
	1259	}
	1260
	1261	if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */
	1262	DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (unsigned)dctx->decodedSize);
	1263	RETURN_ERROR_IF(
	1264	dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
	1265	&& dctx->decodedSize != dctx->fParams.frameContentSize,
	1266	corruption_detected, "");
	1267	if (dctx->fParams.checksumFlag) { /* another round for frame checksum */
	1268	dctx->expected = 4;
	1269	dctx->stage = ZSTDds_checkChecksum;
	1270	} else {
	1271	ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1);
	1272	dctx->expected = 0; /* ends here */
	1273	dctx->stage = ZSTDds_getFrameHeaderSize;
	1274	}
	1275	} else {
	1276	dctx->stage = ZSTDds_decodeBlockHeader;
	1277	dctx->expected = ZSTD_blockHeaderSize;
	1278	}
	1279	return rSize;
	1280	}
	1281
	1282	case ZSTDds_checkChecksum:
	1283	assert(srcSize == 4); /* guaranteed by dctx->expected */
	1284	{
	1285	if (dctx->validateChecksum) {
	1286	U32 const h32 = (U32)XXH64_digest(&dctx->xxhState);
	1287	U32 const check32 = MEM_readLE32(src);
	1288	DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32);
	1289	RETURN_ERROR_IF(check32 != h32, checksum_wrong, "");
	1290	}
	1291	ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1);
	1292	dctx->expected = 0;
	1293	dctx->stage = ZSTDds_getFrameHeaderSize;
	1294	return 0;
	1295	}
	1296
	1297	case ZSTDds_decodeSkippableHeader:
	1298	assert(src != NULL);
	1299	assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE);
	1300	ZSTD_memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */
	1301	dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */
	1302	dctx->stage = ZSTDds_skipFrame;
	1303	return 0;
	1304
	1305	case ZSTDds_skipFrame:
	1306	dctx->expected = 0;
	1307	dctx->stage = ZSTDds_getFrameHeaderSize;
	1308	return 0;
	1309
	1310	default:
	1311	assert(0); /* impossible */
	1312	RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */
	1313	}
	1314	}
	1315
	1316
	1317	static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
	1318	{
	1319	dctx->dictEnd = dctx->previousDstEnd;
	1320	dctx->virtualStart = (const char)dict - ((const char)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart));
	1321	dctx->prefixStart = dict;
	1322	dctx->previousDstEnd = (const char*)dict + dictSize;
	1323	#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
	1324	dctx->dictContentBeginForFuzzing = dctx->prefixStart;
	1325	dctx->dictContentEndForFuzzing = dctx->previousDstEnd;
	1326	#endif
	1327	return 0;
	1328	}
	1329
	1330	/*! ZSTD_loadDEntropy() :
	1331	* dict : must point at beginning of a valid zstd dictionary.
	1332	* @return : size of entropy tables read */
	1333	size_t
	1334	ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
	1335	const void* const dict, size_t const dictSize)
	1336	{
	1337	const BYTE* dictPtr = (const BYTE*)dict;
	1338	const BYTE* const dictEnd = dictPtr + dictSize;
	1339
	1340	RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted, "dict is too small");
	1341	assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY); /* dict must be valid */
	1342	dictPtr += 8; /* skip header = magic + dictID */
	1343
	1344	ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, OFTable) == offsetof(ZSTD_entropyDTables_t, LLTable) + sizeof(entropy->LLTable));
	1345	ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, MLTable) == offsetof(ZSTD_entropyDTables_t, OFTable) + sizeof(entropy->OFTable));
	1346	ZSTD_STATIC_ASSERT(sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable) >= HUF_DECOMPRESS_WORKSPACE_SIZE);
	1347	{ void* const workspace = &entropy->LLTable; /* use fse tables as temporary workspace; implies fse tables are grouped together */
	1348	size_t const workspaceSize = sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable);
	1349	#ifdef HUF_FORCE_DECOMPRESS_X1
	1350	/* in minimal huffman, we always use X1 variants */
	1351	size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable,
	1352	dictPtr, dictEnd - dictPtr,
	1353	workspace, workspaceSize);
	1354	#else
	1355	size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable,
	1356	dictPtr, (size_t)(dictEnd - dictPtr),
	1357	workspace, workspaceSize);
	1358	#endif
	1359	RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, "");
	1360	dictPtr += hSize;
	1361	}
	1362
	1363	{ short offcodeNCount[MaxOff+1];
	1364	unsigned offcodeMaxValue = MaxOff, offcodeLog;
	1365	size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, (size_t)(dictEnd-dictPtr));
	1366	RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, "");
	1367	RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted, "");
	1368	RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, "");
	1369	ZSTD_buildFSETable( entropy->OFTable,
	1370	offcodeNCount, offcodeMaxValue,
	1371	OF_base, OF_bits,
	1372	offcodeLog,
	1373	entropy->workspace, sizeof(entropy->workspace),
	1374	/* bmi2 */0);
	1375	dictPtr += offcodeHeaderSize;
	1376	}
	1377
	1378	{ short matchlengthNCount[MaxML+1];
	1379	unsigned matchlengthMaxValue = MaxML, matchlengthLog;
	1380	size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, (size_t)(dictEnd-dictPtr));
	1381	RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, "");
	1382	RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted, "");
	1383	RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, "");
	1384	ZSTD_buildFSETable( entropy->MLTable,
	1385	matchlengthNCount, matchlengthMaxValue,
	1386	ML_base, ML_bits,
	1387	matchlengthLog,
	1388	entropy->workspace, sizeof(entropy->workspace),
	1389	/* bmi2 */ 0);
	1390	dictPtr += matchlengthHeaderSize;
	1391	}
	1392
	1393	{ short litlengthNCount[MaxLL+1];
	1394	unsigned litlengthMaxValue = MaxLL, litlengthLog;
	1395	size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, (size_t)(dictEnd-dictPtr));
	1396	RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, "");
	1397	RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted, "");
	1398	RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, "");
	1399	ZSTD_buildFSETable( entropy->LLTable,
	1400	litlengthNCount, litlengthMaxValue,
	1401	LL_base, LL_bits,
	1402	litlengthLog,
	1403	entropy->workspace, sizeof(entropy->workspace),
	1404	/* bmi2 */ 0);
	1405	dictPtr += litlengthHeaderSize;
	1406	}
	1407
	1408	RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, "");
	1409	{ int i;
	1410	size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12));
	1411	for (i=0; i<3; i++) {
	1412	U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4;
	1413	RETURN_ERROR_IF(rep==0 \|\| rep > dictContentSize,
	1414	dictionary_corrupted, "");
	1415	entropy->rep[i] = rep;
	1416	} }
	1417
	1418	return (size_t)(dictPtr - (const BYTE*)dict);
	1419	}
	1420
	1421	static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
	1422	{
	1423	if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize);
	1424	{ U32 const magic = MEM_readLE32(dict);
	1425	if (magic != ZSTD_MAGIC_DICTIONARY) {
	1426	return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */
	1427	} }
	1428	dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE);
	1429
	1430	/* load entropy tables */
	1431	{ size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize);
	1432	RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted, "");
	1433	dict = (const char*)dict + eSize;
	1434	dictSize -= eSize;
	1435	}
	1436	dctx->litEntropy = dctx->fseEntropy = 1;
	1437
	1438	/* reference dictionary content */
	1439	return ZSTD_refDictContent(dctx, dict, dictSize);
	1440	}
	1441
	1442	size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
	1443	{
	1444	assert(dctx != NULL);
	1445	#if ZSTD_TRACE
	1446	dctx->traceCtx = (ZSTD_trace_decompress_begin != NULL) ? ZSTD_trace_decompress_begin(dctx) : 0;
	1447	#endif
	1448	dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */
	1449	dctx->stage = ZSTDds_getFrameHeaderSize;
	1450	dctx->processedCSize = 0;
	1451	dctx->decodedSize = 0;
	1452	dctx->previousDstEnd = NULL;
	1453	dctx->prefixStart = NULL;
	1454	dctx->virtualStart = NULL;
	1455	dctx->dictEnd = NULL;
	1456	dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)0x1000001); / cover both little and big endian */
	1457	dctx->litEntropy = dctx->fseEntropy = 0;
	1458	dctx->dictID = 0;
	1459	dctx->bType = bt_reserved;
	1460	ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue));
	1461	ZSTD_memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */
	1462	dctx->LLTptr = dctx->entropy.LLTable;
	1463	dctx->MLTptr = dctx->entropy.MLTable;
	1464	dctx->OFTptr = dctx->entropy.OFTable;
	1465	dctx->HUFptr = dctx->entropy.hufTable;
	1466	return 0;
	1467	}
	1468
	1469	size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
	1470	{
	1471	FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , "");
	1472	if (dict && dictSize)
	1473	RETURN_ERROR_IF(
	1474	ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)),
	1475	dictionary_corrupted, "");
	1476	return 0;
	1477	}
	1478
	1479
	1480	/* ====== ZSTD_DDict ====== */
	1481
	1482	size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
	1483	{
	1484	DEBUGLOG(4, "ZSTD_decompressBegin_usingDDict");
	1485	assert(dctx != NULL);
	1486	if (ddict) {
	1487	const char* const dictStart = (const char*)ZSTD_DDict_dictContent(ddict);
	1488	size_t const dictSize = ZSTD_DDict_dictSize(ddict);
	1489	const void* const dictEnd = dictStart + dictSize;
	1490	dctx->ddictIsCold = (dctx->dictEnd != dictEnd);
	1491	DEBUGLOG(4, "DDict is %s",
	1492	dctx->ddictIsCold ? "~cold~" : "hot!");
	1493	}
	1494	FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , "");
	1495	if (ddict) { /* NULL ddict is equivalent to no dictionary */
	1496	ZSTD_copyDDictParameters(dctx, ddict);
	1497	}
	1498	return 0;
	1499	}
	1500
	1501	/*! ZSTD_getDictID_fromDict() :
	1502	* Provides the dictID stored within dictionary.
	1503	* if @return == 0, the dictionary is not conformant with Zstandard specification.
	1504	* It can still be loaded, but as a content-only dictionary. */
	1505	unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
	1506	{
	1507	if (dictSize < 8) return 0;
	1508	if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0;
	1509	return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE);
	1510	}
	1511
	1512	/*! ZSTD_getDictID_fromFrame() :
	1513	* Provides the dictID required to decompress frame stored within `src`.
	1514	* If @return == 0, the dictID could not be decoded.
	1515	* This could for one of the following reasons :
	1516	* - The frame does not require a dictionary (most common case).
	1517	* - The frame was built with dictID intentionally removed.
	1518	* Needed dictionary is a hidden information.
	1519	* Note : this use case also happens when using a non-conformant dictionary.
	1520	* - `srcSize` is too small, and as a result, frame header could not be decoded.
	1521	* Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`.
	1522	* - This is not a Zstandard frame.
	1523	* When identifying the exact failure cause, it's possible to use
	1524	* ZSTD_getFrameHeader(), which will provide a more precise error code. */
	1525	unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
	1526	{
	1527	ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0 };
	1528	size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize);
	1529	if (ZSTD_isError(hError)) return 0;
	1530	return zfp.dictID;
	1531	}
	1532
	1533
	1534	/*! ZSTD_decompress_usingDDict() :
	1535	* Decompression using a pre-digested Dictionary
	1536	* Use dictionary without significant overhead. */
	1537	size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
	1538	void* dst, size_t dstCapacity,
	1539	const void* src, size_t srcSize,
	1540	const ZSTD_DDict* ddict)
	1541	{
	1542	/* pass content and size in case legacy frames are encountered */
	1543	return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize,
	1544	NULL, 0,
	1545	ddict);
	1546	}
	1547
	1548
	1549	/*=====================================
	1550	* Streaming decompression
	1551	====================================/
	1552
	1553	ZSTD_DStream* ZSTD_createDStream(void)
	1554	{
	1555	DEBUGLOG(3, "ZSTD_createDStream");
	1556	return ZSTD_createDCtx_internal(ZSTD_defaultCMem);
	1557	}
	1558
	1559	ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize)
	1560	{
	1561	return ZSTD_initStaticDCtx(workspace, workspaceSize);
	1562	}
	1563
	1564	ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
	1565	{
	1566	return ZSTD_createDCtx_internal(customMem);
	1567	}
	1568
	1569	size_t ZSTD_freeDStream(ZSTD_DStream* zds)
	1570	{
	1571	return ZSTD_freeDCtx(zds);
	1572	}
	1573
	1574
	1575	/* * Initialization * */
	1576
	1577	size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; }
	1578	size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; }
	1579
	1580	size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx,
	1581	const void* dict, size_t dictSize,
	1582	ZSTD_dictLoadMethod_e dictLoadMethod,
	1583	ZSTD_dictContentType_e dictContentType)
	1584	{
	1585	RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
	1586	ZSTD_clearDict(dctx);
	1587	if (dict && dictSize != 0) {
	1588	dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem);
	1589	RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation, "NULL pointer!");
	1590	dctx->ddict = dctx->ddictLocal;
	1591	dctx->dictUses = ZSTD_use_indefinitely;
	1592	}
	1593	return 0;
	1594	}
	1595
	1596	size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
	1597	{
	1598	return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto);
	1599	}
	1600
	1601	size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
	1602	{
	1603	return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto);
	1604	}
	1605
	1606	size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType)
	1607	{
	1608	FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType), "");
	1609	dctx->dictUses = ZSTD_use_once;
	1610	return 0;
	1611	}
	1612
	1613	size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize)
	1614	{
	1615	return ZSTD_DCtx_refPrefix_advanced(dctx, prefix, prefixSize, ZSTD_dct_rawContent);
	1616	}
	1617
	1618
	1619	/* ZSTD_initDStream_usingDict() :
	1620	* return : expected size, aka ZSTD_startingInputLength().
	1621	* this function cannot fail */
	1622	size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize)
	1623	{
	1624	DEBUGLOG(4, "ZSTD_initDStream_usingDict");
	1625	FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) , "");
	1626	FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) , "");
	1627	return ZSTD_startingInputLength(zds->format);
	1628	}
	1629
	1630	/* note : this variant can't fail */
	1631	size_t ZSTD_initDStream(ZSTD_DStream* zds)
	1632	{
	1633	DEBUGLOG(4, "ZSTD_initDStream");
	1634	return ZSTD_initDStream_usingDDict(zds, NULL);
	1635	}
	1636
	1637	/* ZSTD_initDStream_usingDDict() :
	1638	* ddict will just be referenced, and must outlive decompression session
	1639	* this function cannot fail */
	1640	size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
	1641	{
	1642	FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , "");
	1643	FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , "");
	1644	return ZSTD_startingInputLength(dctx->format);
	1645	}
	1646
	1647	/* ZSTD_resetDStream() :
	1648	* return : expected size, aka ZSTD_startingInputLength().
	1649	* this function cannot fail */
	1650	size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
	1651	{
	1652	FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), "");
	1653	return ZSTD_startingInputLength(dctx->format);
	1654	}
	1655
	1656
	1657	size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
	1658	{
	1659	RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
	1660	ZSTD_clearDict(dctx);
	1661	if (ddict) {
	1662	dctx->ddict = ddict;
	1663	dctx->dictUses = ZSTD_use_indefinitely;
	1664	if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts) {
	1665	if (dctx->ddictSet == NULL) {
	1666	dctx->ddictSet = ZSTD_createDDictHashSet(dctx->customMem);
	1667	if (!dctx->ddictSet) {
	1668	RETURN_ERROR(memory_allocation, "Failed to allocate memory for hash set!");
	1669	}
	1670	}
	1671	assert(!dctx->staticSize); /* Impossible: ddictSet cannot have been allocated if static dctx */
	1672	FORWARD_IF_ERROR(ZSTD_DDictHashSet_addDDict(dctx->ddictSet, ddict, dctx->customMem), "");
	1673	}
	1674	}
	1675	return 0;
	1676	}
	1677
	1678	/* ZSTD_DCtx_setMaxWindowSize() :
	1679	* note : no direct equivalence in ZSTD_DCtx_setParameter,
	1680	* since this version sets windowSize, and the other sets windowLog */
	1681	size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize)
	1682	{
	1683	ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
	1684	size_t const min = (size_t)1 << bounds.lowerBound;
	1685	size_t const max = (size_t)1 << bounds.upperBound;
	1686	RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
	1687	RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound, "");
	1688	RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound, "");
	1689	dctx->maxWindowSize = maxWindowSize;
	1690	return 0;
	1691	}
	1692
	1693	size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format)
	1694	{
	1695	return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (int)format);
	1696	}
	1697
	1698	ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam)
	1699	{
	1700	ZSTD_bounds bounds = { 0, 0, 0 };
	1701	switch(dParam) {
	1702	case ZSTD_d_windowLogMax:
	1703	bounds.lowerBound = ZSTD_WINDOWLOG_ABSOLUTEMIN;
	1704	bounds.upperBound = ZSTD_WINDOWLOG_MAX;
	1705	return bounds;
	1706	case ZSTD_d_format:
	1707	bounds.lowerBound = (int)ZSTD_f_zstd1;
	1708	bounds.upperBound = (int)ZSTD_f_zstd1_magicless;
	1709	ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless);
	1710	return bounds;
	1711	case ZSTD_d_stableOutBuffer:
	1712	bounds.lowerBound = (int)ZSTD_bm_buffered;
	1713	bounds.upperBound = (int)ZSTD_bm_stable;
	1714	return bounds;
	1715	case ZSTD_d_forceIgnoreChecksum:
	1716	bounds.lowerBound = (int)ZSTD_d_validateChecksum;
	1717	bounds.upperBound = (int)ZSTD_d_ignoreChecksum;
	1718	return bounds;
	1719	case ZSTD_d_refMultipleDDicts:
	1720	bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict;
	1721	bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts;
	1722	return bounds;
	1723	default:;
	1724	}
	1725	bounds.error = ERROR(parameter_unsupported);
	1726	return bounds;
	1727	}
	1728
	1729	/* ZSTD_dParam_withinBounds:
	1730	* @return 1 if value is within dParam bounds,
	1731	* 0 otherwise */
	1732	static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value)
	1733	{
	1734	ZSTD_bounds const bounds = ZSTD_dParam_getBounds(dParam);
	1735	if (ZSTD_isError(bounds.error)) return 0;
	1736	if (value < bounds.lowerBound) return 0;
	1737	if (value > bounds.upperBound) return 0;
	1738	return 1;
	1739	}
	1740
	1741	#define CHECK_DBOUNDS(p,v) { \
	1742	RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound, ""); \
	1743	}
	1744
	1745	size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value)
	1746	{
	1747	switch (param) {
	1748	case ZSTD_d_windowLogMax:
	1749	*value = (int)ZSTD_highbit32((U32)dctx->maxWindowSize);
	1750	return 0;
	1751	case ZSTD_d_format:
	1752	*value = (int)dctx->format;
	1753	return 0;
	1754	case ZSTD_d_stableOutBuffer:
	1755	*value = (int)dctx->outBufferMode;
	1756	return 0;
	1757	case ZSTD_d_forceIgnoreChecksum:
	1758	*value = (int)dctx->forceIgnoreChecksum;
	1759	return 0;
	1760	case ZSTD_d_refMultipleDDicts:
	1761	*value = (int)dctx->refMultipleDDicts;
	1762	return 0;
	1763	default:;
	1764	}
	1765	RETURN_ERROR(parameter_unsupported, "");
	1766	}
	1767
	1768	size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value)
	1769	{
	1770	RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
	1771	switch(dParam) {
	1772	case ZSTD_d_windowLogMax:
	1773	if (value == 0) value = ZSTD_WINDOWLOG_LIMIT_DEFAULT;
	1774	CHECK_DBOUNDS(ZSTD_d_windowLogMax, value);
	1775	dctx->maxWindowSize = ((size_t)1) << value;
	1776	return 0;
	1777	case ZSTD_d_format:
	1778	CHECK_DBOUNDS(ZSTD_d_format, value);
	1779	dctx->format = (ZSTD_format_e)value;
	1780	return 0;
	1781	case ZSTD_d_stableOutBuffer:
	1782	CHECK_DBOUNDS(ZSTD_d_stableOutBuffer, value);
	1783	dctx->outBufferMode = (ZSTD_bufferMode_e)value;
	1784	return 0;
	1785	case ZSTD_d_forceIgnoreChecksum:
	1786	CHECK_DBOUNDS(ZSTD_d_forceIgnoreChecksum, value);
	1787	dctx->forceIgnoreChecksum = (ZSTD_forceIgnoreChecksum_e)value;
	1788	return 0;
	1789	case ZSTD_d_refMultipleDDicts:
	1790	CHECK_DBOUNDS(ZSTD_d_refMultipleDDicts, value);
	1791	if (dctx->staticSize != 0) {
	1792	RETURN_ERROR(parameter_unsupported, "Static dctx does not support multiple DDicts!");
	1793	}
	1794	dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value;
	1795	return 0;
	1796	default:;
	1797	}
	1798	RETURN_ERROR(parameter_unsupported, "");
	1799	}
	1800
	1801	size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset)
	1802	{
	1803	if ( (reset == ZSTD_reset_session_only)
	1804	\|\| (reset == ZSTD_reset_session_and_parameters) ) {
	1805	dctx->streamStage = zdss_init;
	1806	dctx->noForwardProgress = 0;
	1807	}
	1808	if ( (reset == ZSTD_reset_parameters)
	1809	\|\| (reset == ZSTD_reset_session_and_parameters) ) {
	1810	RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, "");
	1811	ZSTD_clearDict(dctx);
	1812	ZSTD_DCtx_resetParameters(dctx);
	1813	}
	1814	return 0;
	1815	}
	1816
	1817
	1818	size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx)
	1819	{
	1820	return ZSTD_sizeof_DCtx(dctx);
	1821	}
	1822
	1823	size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize)
	1824	{
	1825	size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
	1826	/* space is needed to store the litbuffer after the output of a given block without stomping the extDict of a previous run, as well as to cover both windows against wildcopy*/
	1827	unsigned long long const neededRBSize = windowSize + blockSize + ZSTD_BLOCKSIZE_MAX + (WILDCOPY_OVERLENGTH * 2);
	1828	unsigned long long const neededSize = MIN(frameContentSize, neededRBSize);
	1829	size_t const minRBSize = (size_t) neededSize;
	1830	RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize,
	1831	frameParameter_windowTooLarge, "");
	1832	return minRBSize;
	1833	}
	1834
	1835	size_t ZSTD_estimateDStreamSize(size_t windowSize)
	1836	{
	1837	size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
	1838	size_t const inBuffSize = blockSize; /* no block can be larger */
	1839	size_t const outBuffSize = ZSTD_decodingBufferSize_min(windowSize, ZSTD_CONTENTSIZE_UNKNOWN);
	1840	return ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize;
	1841	}
	1842
	1843	size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize)
	1844	{
	1845	U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */
	1846	ZSTD_frameHeader zfh;
	1847	size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize);
	1848	if (ZSTD_isError(err)) return err;
	1849	RETURN_ERROR_IF(err>0, srcSize_wrong, "");
	1850	RETURN_ERROR_IF(zfh.windowSize > windowSizeMax,
	1851	frameParameter_windowTooLarge, "");
	1852	return ZSTD_estimateDStreamSize((size_t)zfh.windowSize);
	1853	}
	1854
	1855
	1856	/* *** Decompression *** */
	1857
	1858	static int ZSTD_DCtx_isOverflow(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize)
	1859	{
	1860	return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_WORKSPACETOOLARGE_FACTOR;
	1861	}
	1862
	1863	static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize)
	1864	{
	1865	if (ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize))
	1866	zds->oversizedDuration++;
	1867	else
	1868	zds->oversizedDuration = 0;
	1869	}
	1870
	1871	static int ZSTD_DCtx_isOversizedTooLong(ZSTD_DStream* zds)
	1872	{
	1873	return zds->oversizedDuration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION;
	1874	}
	1875
	1876	/* Checks that the output buffer hasn't changed if ZSTD_obm_stable is used. */
	1877	static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const* output)
	1878	{
	1879	ZSTD_outBuffer const expect = zds->expectedOutBuffer;
	1880	/* No requirement when ZSTD_obm_stable is not enabled. */
	1881	if (zds->outBufferMode != ZSTD_bm_stable)
	1882	return 0;
	1883	/* Any buffer is allowed in zdss_init, this must be the same for every other call until
	1884	* the context is reset.
	1885	*/
	1886	if (zds->streamStage == zdss_init)
	1887	return 0;
	1888	/* The buffer must match our expectation exactly. */
	1889	if (expect.dst == output->dst && expect.pos == output->pos && expect.size == output->size)
	1890	return 0;
	1891	RETURN_ERROR(dstBuffer_wrong, "ZSTD_d_stableOutBuffer enabled but output differs!");
	1892	}
	1893
	1894	/* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream()
	1895	* and updates the stage and the output buffer state. This call is extracted so it can be
	1896	* used both when reading directly from the ZSTD_inBuffer, and in buffered input mode.
	1897	* NOTE: You must break after calling this function since the streamStage is modified.
	1898	*/
	1899	static size_t ZSTD_decompressContinueStream(
	1900	ZSTD_DStream* zds, char** op, char* oend,
	1901	void const* src, size_t srcSize) {
	1902	int const isSkipFrame = ZSTD_isSkipFrame(zds);
	1903	if (zds->outBufferMode == ZSTD_bm_buffered) {
	1904	size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart;
	1905	size_t const decodedSize = ZSTD_decompressContinue(zds,
	1906	zds->outBuff + zds->outStart, dstSize, src, srcSize);
	1907	FORWARD_IF_ERROR(decodedSize, "");
	1908	if (!decodedSize && !isSkipFrame) {
	1909	zds->streamStage = zdss_read;
	1910	} else {
	1911	zds->outEnd = zds->outStart + decodedSize;
	1912	zds->streamStage = zdss_flush;
	1913	}
	1914	} else {
	1915	/* Write directly into the output buffer */
	1916	size_t const dstSize = isSkipFrame ? 0 : (size_t)(oend - *op);
	1917	size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize);
	1918	FORWARD_IF_ERROR(decodedSize, "");
	1919	*op += decodedSize;
	1920	/* Flushing is not needed. */
	1921	zds->streamStage = zdss_read;
	1922	assert(*op <= oend);
	1923	assert(zds->outBufferMode == ZSTD_bm_stable);
	1924	}
	1925	return 0;
	1926	}
	1927
	1928	size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input)
	1929	{
	1930	const char* const src = (const char*)input->src;
	1931	const char* const istart = input->pos != 0 ? src + input->pos : src;
	1932	const char* const iend = input->size != 0 ? src + input->size : src;
	1933	const char* ip = istart;
	1934	char* const dst = (char*)output->dst;
	1935	char* const ostart = output->pos != 0 ? dst + output->pos : dst;
	1936	char* const oend = output->size != 0 ? dst + output->size : dst;
	1937	char* op = ostart;
	1938	U32 someMoreWork = 1;
	1939
	1940	DEBUGLOG(5, "ZSTD_decompressStream");
	1941	RETURN_ERROR_IF(
	1942	input->pos > input->size,
	1943	srcSize_wrong,
	1944	"forbidden. in: pos: %u vs size: %u",
	1945	(U32)input->pos, (U32)input->size);
	1946	RETURN_ERROR_IF(
	1947	output->pos > output->size,
	1948	dstSize_tooSmall,
	1949	"forbidden. out: pos: %u vs size: %u",
	1950	(U32)output->pos, (U32)output->size);
	1951	DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos));
	1952	FORWARD_IF_ERROR(ZSTD_checkOutBuffer(zds, output), "");
	1953
	1954	while (someMoreWork) {
	1955	switch(zds->streamStage)
	1956	{
	1957	case zdss_init :
	1958	DEBUGLOG(5, "stage zdss_init => transparent reset ");
	1959	zds->streamStage = zdss_loadHeader;
	1960	zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
	1961	#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
	1962	zds->legacyVersion = 0;
	1963	#endif
	1964	zds->hostageByte = 0;
	1965	zds->expectedOutBuffer = *output;
	1966	ZSTD_FALLTHROUGH;
	1967
	1968	case zdss_loadHeader :
	1969	DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip));
	1970	#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
	1971	if (zds->legacyVersion) {
	1972	RETURN_ERROR_IF(zds->staticSize, memory_allocation,
	1973	"legacy support is incompatible with static dctx");
	1974	{ size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
	1975	if (hint==0) zds->streamStage = zdss_init;
	1976	return hint;
	1977	} }
	1978	#endif
	1979	{ size_t const hSize = ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format);
	1980	if (zds->refMultipleDDicts && zds->ddictSet) {
	1981	ZSTD_DCtx_selectFrameDDict(zds);
	1982	}
	1983	DEBUGLOG(5, "header size : %u", (U32)hSize);
	1984	if (ZSTD_isError(hSize)) {
	1985	#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
	1986	U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
	1987	if (legacyVersion) {
	1988	ZSTD_DDict const* const ddict = ZSTD_getDDict(zds);
	1989	const void* const dict = ddict ? ZSTD_DDict_dictContent(ddict) : NULL;
	1990	size_t const dictSize = ddict ? ZSTD_DDict_dictSize(ddict) : 0;
	1991	DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion);
	1992	RETURN_ERROR_IF(zds->staticSize, memory_allocation,
	1993	"legacy support is incompatible with static dctx");
	1994	FORWARD_IF_ERROR(ZSTD_initLegacyStream(&zds->legacyContext,
	1995	zds->previousLegacyVersion, legacyVersion,
	1996	dict, dictSize), "");
	1997	zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
	1998	{ size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input);
	1999	if (hint==0) zds->streamStage = zdss_init; /* or stay in stage zdss_loadHeader */
	2000	return hint;
	2001	} }
	2002	#endif
	2003	return hSize; /* error */
	2004	}
	2005	if (hSize != 0) { /* need more input */
	2006	size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */
	2007	size_t const remainingInput = (size_t)(iend-ip);
	2008	assert(iend >= ip);
	2009	if (toLoad > remainingInput) { /* not enough input to load full header */
	2010	if (remainingInput > 0) {
	2011	ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput);
	2012	zds->lhSize += remainingInput;
	2013	}
	2014	input->pos = input->size;
	2015	return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
	2016	}
	2017	assert(ip != NULL);
	2018	ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;
	2019	break;
	2020	} }
	2021
	2022	/* check for single-pass mode opportunity */
	2023	if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
	2024	&& zds->fParams.frameType != ZSTD_skippableFrame
	2025	&& (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) {
	2026	size_t const cSize = ZSTD_findFrameCompressedSize(istart, (size_t)(iend-istart));
	2027	if (cSize <= (size_t)(iend-istart)) {
	2028	/* shortcut : using single-pass mode */
	2029	size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds));
	2030	if (ZSTD_isError(decompressedSize)) return decompressedSize;
	2031	DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()")
	2032	ip = istart + cSize;
	2033	op += decompressedSize;
	2034	zds->expected = 0;
	2035	zds->streamStage = zdss_init;
	2036	someMoreWork = 0;
	2037	break;
	2038	} }
	2039
	2040	/* Check output buffer is large enough for ZSTD_odm_stable. */
	2041	if (zds->outBufferMode == ZSTD_bm_stable
	2042	&& zds->fParams.frameType != ZSTD_skippableFrame
	2043	&& zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
	2044	&& (U64)(size_t)(oend-op) < zds->fParams.frameContentSize) {
	2045	RETURN_ERROR(dstSize_tooSmall, "ZSTD_obm_stable passed but ZSTD_outBuffer is too small");
	2046	}
	2047
	2048	/* Consume header (see ZSTDds_decodeFrameHeader) */
	2049	DEBUGLOG(4, "Consume header");
	2050	FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), "");
	2051
	2052	if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
	2053	zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE);
	2054	zds->stage = ZSTDds_skipFrame;
	2055	} else {
	2056	FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize), "");
	2057	zds->expected = ZSTD_blockHeaderSize;
	2058	zds->stage = ZSTDds_decodeBlockHeader;
	2059	}
	2060
	2061	/* control buffer memory usage */
	2062	DEBUGLOG(4, "Control max memory usage (%u KB <= max %u KB)",
	2063	(U32)(zds->fParams.windowSize >>10),
	2064	(U32)(zds->maxWindowSize >> 10) );
	2065	zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
	2066	RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize,
	2067	frameParameter_windowTooLarge, "");
	2068
	2069	/* Adapt buffer sizes to frame header instructions */
	2070	{ size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */);
	2071	size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_bm_buffered
	2072	? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize)
	2073	: 0;
	2074
	2075	ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize);
	2076
	2077	{ int const tooSmall = (zds->inBuffSize < neededInBuffSize) \|\| (zds->outBuffSize < neededOutBuffSize);
	2078	int const tooLarge = ZSTD_DCtx_isOversizedTooLong(zds);
	2079
	2080	if (tooSmall \|\| tooLarge) {
	2081	size_t const bufferSize = neededInBuffSize + neededOutBuffSize;
	2082	DEBUGLOG(4, "inBuff : from %u to %u",
	2083	(U32)zds->inBuffSize, (U32)neededInBuffSize);
	2084	DEBUGLOG(4, "outBuff : from %u to %u",
	2085	(U32)zds->outBuffSize, (U32)neededOutBuffSize);
	2086	if (zds->staticSize) { /* static DCtx */
	2087	DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize);
	2088	assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */
	2089	RETURN_ERROR_IF(
	2090	bufferSize > zds->staticSize - sizeof(ZSTD_DCtx),
	2091	memory_allocation, "");
	2092	} else {
	2093	ZSTD_customFree(zds->inBuff, zds->customMem);
	2094	zds->inBuffSize = 0;
	2095	zds->outBuffSize = 0;
	2096	zds->inBuff = (char*)ZSTD_customMalloc(bufferSize, zds->customMem);
	2097	RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation, "");
	2098	}
	2099	zds->inBuffSize = neededInBuffSize;
	2100	zds->outBuff = zds->inBuff + zds->inBuffSize;
	2101	zds->outBuffSize = neededOutBuffSize;
	2102	} } }
	2103	zds->streamStage = zdss_read;
	2104	ZSTD_FALLTHROUGH;
	2105
	2106	case zdss_read:
	2107	DEBUGLOG(5, "stage zdss_read");
	2108	{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip));
	2109	DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize);
	2110	if (neededInSize==0) { /* end of frame */
	2111	zds->streamStage = zdss_init;
	2112	someMoreWork = 0;
	2113	break;
	2114	}
	2115	if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
	2116	FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), "");
	2117	ip += neededInSize;
	2118	/* Function modifies the stage so we must break */
	2119	break;
	2120	} }
	2121	if (ip==iend) { someMoreWork = 0; break; } /* no more input */
	2122	zds->streamStage = zdss_load;
	2123	ZSTD_FALLTHROUGH;
	2124
	2125	case zdss_load:
	2126	{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
	2127	size_t const toLoad = neededInSize - zds->inPos;
	2128	int const isSkipFrame = ZSTD_isSkipFrame(zds);
	2129	size_t loadedSize;
	2130	/* At this point we shouldn't be decompressing a block that we can stream. */
	2131	assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip));
	2132	if (isSkipFrame) {
	2133	loadedSize = MIN(toLoad, (size_t)(iend-ip));
	2134	} else {
	2135	RETURN_ERROR_IF(toLoad > zds->inBuffSize - zds->inPos,
	2136	corruption_detected,
	2137	"should never happen");
	2138	loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip));
	2139	}
	2140	ip += loadedSize;
	2141	zds->inPos += loadedSize;
	2142	if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */
	2143
	2144	/* decode loaded input */
	2145	zds->inPos = 0; /* input is consumed */
	2146	FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, zds->inBuff, neededInSize), "");
	2147	/* Function modifies the stage so we must break */
	2148	break;
	2149	}
	2150	case zdss_flush:
	2151	{ size_t const toFlushSize = zds->outEnd - zds->outStart;
	2152	size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize);
	2153	op += flushedSize;
	2154	zds->outStart += flushedSize;
	2155	if (flushedSize == toFlushSize) { /* flush completed */
	2156	zds->streamStage = zdss_read;
	2157	if ( (zds->outBuffSize < zds->fParams.frameContentSize)
	2158	&& (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) {
	2159	DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)",
	2160	(int)(zds->outBuffSize - zds->outStart),
	2161	(U32)zds->fParams.blockSizeMax);
	2162	zds->outStart = zds->outEnd = 0;
	2163	}
	2164	break;
	2165	} }
	2166	/* cannot complete flush */
	2167	someMoreWork = 0;
	2168	break;
	2169
	2170	default:
	2171	assert(0); /* impossible */
	2172	RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */
	2173	} }
	2174
	2175	/* result */
	2176	input->pos = (size_t)(ip - (const char*)(input->src));
	2177	output->pos = (size_t)(op - (char*)(output->dst));
	2178
	2179	/* Update the expected output buffer for ZSTD_obm_stable. */
	2180	zds->expectedOutBuffer = *output;
	2181
	2182	if ((ip==istart) && (op==ostart)) { /* no forward progress */
	2183	zds->noForwardProgress ++;
	2184	if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) {
	2185	RETURN_ERROR_IF(op==oend, dstSize_tooSmall, "");
	2186	RETURN_ERROR_IF(ip==iend, srcSize_wrong, "");
	2187	assert(0);
	2188	}
	2189	} else {
	2190	zds->noForwardProgress = 0;
	2191	}
	2192	{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds);
	2193	if (!nextSrcSizeHint) { /* frame fully decoded */
	2194	if (zds->outEnd == zds->outStart) { /* output fully flushed */
	2195	if (zds->hostageByte) {
	2196	if (input->pos >= input->size) {
	2197	/* can't release hostage (not present) */
	2198	zds->streamStage = zdss_read;
	2199	return 1;
	2200	}
	2201	input->pos++; /* release hostage */
	2202	} /* zds->hostageByte */
	2203	return 0;
	2204	} /* zds->outEnd == zds->outStart */
	2205	if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */
	2206	input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */
	2207	zds->hostageByte=1;
	2208	}
	2209	return 1;
	2210	} /* nextSrcSizeHint==0 */
	2211	nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block); /* preload header of next block */
	2212	assert(zds->inPos <= nextSrcSizeHint);
	2213	nextSrcSizeHint -= zds->inPos; /* part already loaded*/
	2214	return nextSrcSizeHint;
	2215	}
	2216	}
	2217
	2218	size_t ZSTD_decompressStream_simpleArgs (
	2219	ZSTD_DCtx* dctx,
	2220	void* dst, size_t dstCapacity, size_t* dstPos,
	2221	const void* src, size_t srcSize, size_t* srcPos)
	2222	{
	2223	ZSTD_outBuffer output = { dst, dstCapacity, *dstPos };
	2224	ZSTD_inBuffer input = { src, srcSize, *srcPos };
	2225	/* ZSTD_compress_generic() will check validity of dstPos and srcPos */
	2226	size_t const cErr = ZSTD_decompressStream(dctx, &output, &input);
	2227	*dstPos = output.pos;
	2228	*srcPos = input.pos;
	2229	return cErr;
	2230	}

stage1/zstd/lib/decompress/zstd_decompress_block.c created+2072

...	...	@@ -0,0 +1,2072 @@
	1	/*
	2	* Copyright (c) Yann Collet, Facebook, Inc.
	3	* All rights reserved.
	4	*
	5	* This source code is licensed under both the BSD-style license (found in the
	6	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	7	* in the COPYING file in the root directory of this source tree).
	8	* You may select, at your option, one of the above-listed licenses.
	9	*/
	10
	11	/* zstd_decompress_block :
	12	* this module takes care of decompressing _compressed_ block */
	13
	14	/-******************************************************
	15	* Dependencies
	16	*********************************************************/
	17	#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
	18	#include "../common/compiler.h" /* prefetch */
	19	#include "../common/cpu.h" /* bmi2 */
	20	#include "../common/mem.h" /* low level memory routines */
	21	#define FSE_STATIC_LINKING_ONLY
	22	#include "../common/fse.h"
	23	#define HUF_STATIC_LINKING_ONLY
	24	#include "../common/huf.h"
	25	#include "../common/zstd_internal.h"
	26	#include "zstd_decompress_internal.h" /* ZSTD_DCtx */
	27	#include "zstd_ddict.h" /* ZSTD_DDictDictContent */
	28	#include "zstd_decompress_block.h"
	29
	30	/_******************************************************
	31	* Macros
	32	**********************************************************/
	33
	34	/* These two optional macros force the use one way or another of the two
	35	* ZSTD_decompressSequences implementations. You can't force in both directions
	36	* at the same time.
	37	*/
	38	#if defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
	39	defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
	40	#error "Cannot force the use of the short and the long ZSTD_decompressSequences variants!"
	41	#endif
	42
	43
	44	/_******************************************************
	45	* Memory operations
	46	**********************************************************/
	47	static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); }
	48
	49
	50	/-************************************************************
	51	* Block decoding
	52	***************************************************************/
	53
	54	/*! ZSTD_getcBlockSize() :
	55	* Provides the size of compressed block from block header `src` */
	56	size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
	57	blockProperties_t* bpPtr)
	58	{
	59	RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong, "");
	60
	61	{ U32 const cBlockHeader = MEM_readLE24(src);
	62	U32 const cSize = cBlockHeader >> 3;
	63	bpPtr->lastBlock = cBlockHeader & 1;
	64	bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3);
	65	bpPtr->origSize = cSize; /* only useful for RLE */
	66	if (bpPtr->blockType == bt_rle) return 1;
	67	RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, "");
	68	return cSize;
	69	}
	70	}
	71
	72	/* Allocate buffer for literals, either overlapping current dst, or split between dst and litExtraBuffer, or stored entirely within litExtraBuffer */
	73	static void ZSTD_allocateLiteralsBuffer(ZSTD_DCtx* dctx, void* const dst, const size_t dstCapacity, const size_t litSize,
	74	const streaming_operation streaming, const size_t expectedWriteSize, const unsigned splitImmediately)
	75	{
	76	if (streaming == not_streaming && dstCapacity > ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH + litSize + WILDCOPY_OVERLENGTH)
	77	{
	78	/* room for litbuffer to fit without read faulting */
	79	dctx->litBuffer = (BYTE*)dst + ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH;
	80	dctx->litBufferEnd = dctx->litBuffer + litSize;
	81	dctx->litBufferLocation = ZSTD_in_dst;
	82	}
	83	else if (litSize > ZSTD_LITBUFFEREXTRASIZE)
	84	{
	85	/* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */
	86	if (splitImmediately) {
	87	/* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */
	88	dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH;
	89	dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE;
	90	}
	91	else {
	92	/* initially this will be stored entirely in dst during huffman decoding, it will partially shifted to litExtraBuffer after */
	93	dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize;
	94	dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize;
	95	}
	96	dctx->litBufferLocation = ZSTD_split;
	97	}
	98	else
	99	{
	100	/* fits entirely within litExtraBuffer, so no split is necessary */
	101	dctx->litBuffer = dctx->litExtraBuffer;
	102	dctx->litBufferEnd = dctx->litBuffer + litSize;
	103	dctx->litBufferLocation = ZSTD_not_in_dst;
	104	}
	105	}
	106
	107	/* Hidden declaration for fullbench */
	108	size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
	109	const void* src, size_t srcSize,
	110	void* dst, size_t dstCapacity, const streaming_operation streaming);
	111	/*! ZSTD_decodeLiteralsBlock() :
	112	* Where it is possible to do so without being stomped by the output during decompression, the literals block will be stored
	113	* in the dstBuffer. If there is room to do so, it will be stored in full in the excess dst space after where the current
	114	* block will be output. Otherwise it will be stored at the end of the current dst blockspace, with a small portion being
	115	* stored in dctx->litExtraBuffer to help keep it "ahead" of the current output write.
	116	*
	117	* @return : nb of bytes read from src (< srcSize )
	118	* note : symbol not declared but exposed for fullbench */
	119	size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
	120	const void* src, size_t srcSize, /* note : srcSize < BLOCKSIZE */
	121	void* dst, size_t dstCapacity, const streaming_operation streaming)
	122	{
	123	DEBUGLOG(5, "ZSTD_decodeLiteralsBlock");
	124	RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, "");
	125
	126	{ const BYTE* const istart = (const BYTE*) src;
	127	symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3);
	128
	129	switch(litEncType)
	130	{
	131	case set_repeat:
	132	DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block");
	133	RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, "");
	134	ZSTD_FALLTHROUGH;
	135
	136	case set_compressed:
	137	RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3");
	138	{ size_t lhSize, litSize, litCSize;
	139	U32 singleStream=0;
	140	U32 const lhlCode = (istart[0] >> 2) & 3;
	141	U32 const lhc = MEM_readLE32(istart);
	142	size_t hufSuccess;
	143	size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity);
	144	switch(lhlCode)
	145	{
	146	case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */
	147	/* 2 - 2 - 10 - 10 */
	148	singleStream = !lhlCode;
	149	lhSize = 3;
	150	litSize = (lhc >> 4) & 0x3FF;
	151	litCSize = (lhc >> 14) & 0x3FF;
	152	break;
	153	case 2:
	154	/* 2 - 2 - 14 - 14 */
	155	lhSize = 4;
	156	litSize = (lhc >> 4) & 0x3FFF;
	157	litCSize = lhc >> 18;
	158	break;
	159	case 3:
	160	/* 2 - 2 - 18 - 18 */
	161	lhSize = 5;
	162	litSize = (lhc >> 4) & 0x3FFFF;
	163	litCSize = (lhc >> 22) + ((size_t)istart[4] << 10);
	164	break;
	165	}
	166	RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
	167	RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, "");
	168	RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, "");
	169	RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, "");
	170	ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0);
	171
	172	/* prefetch huffman table if cold */
	173	if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) {
	174	PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable));
	175	}
	176
	177	if (litEncType==set_repeat) {
	178	if (singleStream) {
	179	hufSuccess = HUF_decompress1X_usingDTable_bmi2(
	180	dctx->litBuffer, litSize, istart+lhSize, litCSize,
	181	dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx));
	182	} else {
	183	hufSuccess = HUF_decompress4X_usingDTable_bmi2(
	184	dctx->litBuffer, litSize, istart+lhSize, litCSize,
	185	dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx));
	186	}
	187	} else {
	188	if (singleStream) {
	189	#if defined(HUF_FORCE_DECOMPRESS_X2)
	190	hufSuccess = HUF_decompress1X_DCtx_wksp(
	191	dctx->entropy.hufTable, dctx->litBuffer, litSize,
	192	istart+lhSize, litCSize, dctx->workspace,
	193	sizeof(dctx->workspace));
	194	#else
	195	hufSuccess = HUF_decompress1X1_DCtx_wksp_bmi2(
	196	dctx->entropy.hufTable, dctx->litBuffer, litSize,
	197	istart+lhSize, litCSize, dctx->workspace,
	198	sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx));
	199	#endif
	200	} else {
	201	hufSuccess = HUF_decompress4X_hufOnly_wksp_bmi2(
	202	dctx->entropy.hufTable, dctx->litBuffer, litSize,
	203	istart+lhSize, litCSize, dctx->workspace,
	204	sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx));
	205	}
	206	}
	207	if (dctx->litBufferLocation == ZSTD_split)
	208	{
	209	ZSTD_memcpy(dctx->litExtraBuffer, dctx->litBufferEnd - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE);
	210	ZSTD_memmove(dctx->litBuffer + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH, dctx->litBuffer, litSize - ZSTD_LITBUFFEREXTRASIZE);
	211	dctx->litBuffer += ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH;
	212	dctx->litBufferEnd -= WILDCOPY_OVERLENGTH;
	213	}
	214
	215	RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, "");
	216
	217	dctx->litPtr = dctx->litBuffer;
	218	dctx->litSize = litSize;
	219	dctx->litEntropy = 1;
	220	if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable;
	221	return litCSize + lhSize;
	222	}
	223
	224	case set_basic:
	225	{ size_t litSize, lhSize;
	226	U32 const lhlCode = ((istart[0]) >> 2) & 3;
	227	size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity);
	228	switch(lhlCode)
	229	{
	230	case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */
	231	lhSize = 1;
	232	litSize = istart[0] >> 3;
	233	break;
	234	case 1:
	235	lhSize = 2;
	236	litSize = MEM_readLE16(istart) >> 4;
	237	break;
	238	case 3:
	239	lhSize = 3;
	240	litSize = MEM_readLE24(istart) >> 4;
	241	break;
	242	}
	243
	244	RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
	245	RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, "");
	246	ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1);
	247	if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */
	248	RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, "");
	249	if (dctx->litBufferLocation == ZSTD_split)
	250	{
	251	ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize - ZSTD_LITBUFFEREXTRASIZE);
	252	ZSTD_memcpy(dctx->litExtraBuffer, istart + lhSize + litSize - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE);
	253	}
	254	else
	255	{
	256	ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize);
	257	}
	258	dctx->litPtr = dctx->litBuffer;
	259	dctx->litSize = litSize;
	260	return lhSize+litSize;
	261	}
	262	/* direct reference into compressed stream */
	263	dctx->litPtr = istart+lhSize;
	264	dctx->litSize = litSize;
	265	dctx->litBufferEnd = dctx->litPtr + litSize;
	266	dctx->litBufferLocation = ZSTD_not_in_dst;
	267	return lhSize+litSize;
	268	}
	269
	270	case set_rle:
	271	{ U32 const lhlCode = ((istart[0]) >> 2) & 3;
	272	size_t litSize, lhSize;
	273	size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity);
	274	switch(lhlCode)
	275	{
	276	case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */
	277	lhSize = 1;
	278	litSize = istart[0] >> 3;
	279	break;
	280	case 1:
	281	lhSize = 2;
	282	litSize = MEM_readLE16(istart) >> 4;
	283	break;
	284	case 3:
	285	lhSize = 3;
	286	litSize = MEM_readLE24(istart) >> 4;
	287	RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4");
	288	break;
	289	}
	290	RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
	291	RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, "");
	292	RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, "");
	293	ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1);
	294	if (dctx->litBufferLocation == ZSTD_split)
	295	{
	296	ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize - ZSTD_LITBUFFEREXTRASIZE);
	297	ZSTD_memset(dctx->litExtraBuffer, istart[lhSize], ZSTD_LITBUFFEREXTRASIZE);
	298	}
	299	else
	300	{
	301	ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize);
	302	}
	303	dctx->litPtr = dctx->litBuffer;
	304	dctx->litSize = litSize;
	305	return lhSize+1;
	306	}
	307	default:
	308	RETURN_ERROR(corruption_detected, "impossible");
	309	}
	310	}
	311	}
	312
	313	/* Default FSE distribution tables.
	314	* These are pre-calculated FSE decoding tables using default distributions as defined in specification :
	315	* https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions
	316	* They were generated programmatically with following method :
	317	* - start from default distributions, present in /lib/common/zstd_internal.h
	318	* - generate tables normally, using ZSTD_buildFSETable()
	319	* - printout the content of tables
	320	* - pretify output, report below, test with fuzzer to ensure it's correct */
	321
	322	/* Default FSE distribution table for Literal Lengths */
	323	static const ZSTD_seqSymbol LL_defaultDTable[(1<<LL_DEFAULTNORMLOG)+1] = {
	324	{ 1, 1, 1, LL_DEFAULTNORMLOG}, /* header : fastMode, tableLog */
	325	/* nextState, nbAddBits, nbBits, baseVal */
	326	{ 0, 0, 4, 0}, { 16, 0, 4, 0},
	327	{ 32, 0, 5, 1}, { 0, 0, 5, 3},
	328	{ 0, 0, 5, 4}, { 0, 0, 5, 6},
	329	{ 0, 0, 5, 7}, { 0, 0, 5, 9},
	330	{ 0, 0, 5, 10}, { 0, 0, 5, 12},
	331	{ 0, 0, 6, 14}, { 0, 1, 5, 16},
	332	{ 0, 1, 5, 20}, { 0, 1, 5, 22},
	333	{ 0, 2, 5, 28}, { 0, 3, 5, 32},
	334	{ 0, 4, 5, 48}, { 32, 6, 5, 64},
	335	{ 0, 7, 5, 128}, { 0, 8, 6, 256},
	336	{ 0, 10, 6, 1024}, { 0, 12, 6, 4096},
	337	{ 32, 0, 4, 0}, { 0, 0, 4, 1},
	338	{ 0, 0, 5, 2}, { 32, 0, 5, 4},
	339	{ 0, 0, 5, 5}, { 32, 0, 5, 7},
	340	{ 0, 0, 5, 8}, { 32, 0, 5, 10},
	341	{ 0, 0, 5, 11}, { 0, 0, 6, 13},
	342	{ 32, 1, 5, 16}, { 0, 1, 5, 18},
	343	{ 32, 1, 5, 22}, { 0, 2, 5, 24},
	344	{ 32, 3, 5, 32}, { 0, 3, 5, 40},
	345	{ 0, 6, 4, 64}, { 16, 6, 4, 64},
	346	{ 32, 7, 5, 128}, { 0, 9, 6, 512},
	347	{ 0, 11, 6, 2048}, { 48, 0, 4, 0},
	348	{ 16, 0, 4, 1}, { 32, 0, 5, 2},
	349	{ 32, 0, 5, 3}, { 32, 0, 5, 5},
	350	{ 32, 0, 5, 6}, { 32, 0, 5, 8},
	351	{ 32, 0, 5, 9}, { 32, 0, 5, 11},
	352	{ 32, 0, 5, 12}, { 0, 0, 6, 15},
	353	{ 32, 1, 5, 18}, { 32, 1, 5, 20},
	354	{ 32, 2, 5, 24}, { 32, 2, 5, 28},
	355	{ 32, 3, 5, 40}, { 32, 4, 5, 48},
	356	{ 0, 16, 6,65536}, { 0, 15, 6,32768},
	357	{ 0, 14, 6,16384}, { 0, 13, 6, 8192},
	358	}; /* LL_defaultDTable */
	359
	360	/* Default FSE distribution table for Offset Codes */
	361	static const ZSTD_seqSymbol OF_defaultDTable[(1<<OF_DEFAULTNORMLOG)+1] = {
	362	{ 1, 1, 1, OF_DEFAULTNORMLOG}, /* header : fastMode, tableLog */
	363	/* nextState, nbAddBits, nbBits, baseVal */
	364	{ 0, 0, 5, 0}, { 0, 6, 4, 61},
	365	{ 0, 9, 5, 509}, { 0, 15, 5,32765},
	366	{ 0, 21, 5,2097149}, { 0, 3, 5, 5},
	367	{ 0, 7, 4, 125}, { 0, 12, 5, 4093},
	368	{ 0, 18, 5,262141}, { 0, 23, 5,8388605},
	369	{ 0, 5, 5, 29}, { 0, 8, 4, 253},
	370	{ 0, 14, 5,16381}, { 0, 20, 5,1048573},
	371	{ 0, 2, 5, 1}, { 16, 7, 4, 125},
	372	{ 0, 11, 5, 2045}, { 0, 17, 5,131069},
	373	{ 0, 22, 5,4194301}, { 0, 4, 5, 13},
	374	{ 16, 8, 4, 253}, { 0, 13, 5, 8189},
	375	{ 0, 19, 5,524285}, { 0, 1, 5, 1},
	376	{ 16, 6, 4, 61}, { 0, 10, 5, 1021},
	377	{ 0, 16, 5,65533}, { 0, 28, 5,268435453},
	378	{ 0, 27, 5,134217725}, { 0, 26, 5,67108861},
	379	{ 0, 25, 5,33554429}, { 0, 24, 5,16777213},
	380	}; /* OF_defaultDTable */
	381
	382
	383	/* Default FSE distribution table for Match Lengths */
	384	static const ZSTD_seqSymbol ML_defaultDTable[(1<<ML_DEFAULTNORMLOG)+1] = {
	385	{ 1, 1, 1, ML_DEFAULTNORMLOG}, /* header : fastMode, tableLog */
	386	/* nextState, nbAddBits, nbBits, baseVal */
	387	{ 0, 0, 6, 3}, { 0, 0, 4, 4},
	388	{ 32, 0, 5, 5}, { 0, 0, 5, 6},
	389	{ 0, 0, 5, 8}, { 0, 0, 5, 9},
	390	{ 0, 0, 5, 11}, { 0, 0, 6, 13},
	391	{ 0, 0, 6, 16}, { 0, 0, 6, 19},
	392	{ 0, 0, 6, 22}, { 0, 0, 6, 25},
	393	{ 0, 0, 6, 28}, { 0, 0, 6, 31},
	394	{ 0, 0, 6, 34}, { 0, 1, 6, 37},
	395	{ 0, 1, 6, 41}, { 0, 2, 6, 47},
	396	{ 0, 3, 6, 59}, { 0, 4, 6, 83},
	397	{ 0, 7, 6, 131}, { 0, 9, 6, 515},
	398	{ 16, 0, 4, 4}, { 0, 0, 4, 5},
	399	{ 32, 0, 5, 6}, { 0, 0, 5, 7},
	400	{ 32, 0, 5, 9}, { 0, 0, 5, 10},
	401	{ 0, 0, 6, 12}, { 0, 0, 6, 15},
	402	{ 0, 0, 6, 18}, { 0, 0, 6, 21},
	403	{ 0, 0, 6, 24}, { 0, 0, 6, 27},
	404	{ 0, 0, 6, 30}, { 0, 0, 6, 33},
	405	{ 0, 1, 6, 35}, { 0, 1, 6, 39},
	406	{ 0, 2, 6, 43}, { 0, 3, 6, 51},
	407	{ 0, 4, 6, 67}, { 0, 5, 6, 99},
	408	{ 0, 8, 6, 259}, { 32, 0, 4, 4},
	409	{ 48, 0, 4, 4}, { 16, 0, 4, 5},
	410	{ 32, 0, 5, 7}, { 32, 0, 5, 8},
	411	{ 32, 0, 5, 10}, { 32, 0, 5, 11},
	412	{ 0, 0, 6, 14}, { 0, 0, 6, 17},
	413	{ 0, 0, 6, 20}, { 0, 0, 6, 23},
	414	{ 0, 0, 6, 26}, { 0, 0, 6, 29},
	415	{ 0, 0, 6, 32}, { 0, 16, 6,65539},
	416	{ 0, 15, 6,32771}, { 0, 14, 6,16387},
	417	{ 0, 13, 6, 8195}, { 0, 12, 6, 4099},
	418	{ 0, 11, 6, 2051}, { 0, 10, 6, 1027},
	419	}; /* ML_defaultDTable */
	420
	421
	422	static void ZSTD_buildSeqTable_rle(ZSTD_seqSymbol* dt, U32 baseValue, U8 nbAddBits)
	423	{
	424	void* ptr = dt;
	425	ZSTD_seqSymbol_header* const DTableH = (ZSTD_seqSymbol_header*)ptr;
	426	ZSTD_seqSymbol* const cell = dt + 1;
	427
	428	DTableH->tableLog = 0;
	429	DTableH->fastMode = 0;
	430
	431	cell->nbBits = 0;
	432	cell->nextState = 0;
	433	assert(nbAddBits < 255);
	434	cell->nbAdditionalBits = nbAddBits;
	435	cell->baseValue = baseValue;
	436	}
	437
	438
	439	/* ZSTD_buildFSETable() :
	440	* generate FSE decoding table for one symbol (ll, ml or off)
	441	* cannot fail if input is valid =>
	442	* all inputs are presumed validated at this stage */
	443	FORCE_INLINE_TEMPLATE
	444	void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt,
	445	const short* normalizedCounter, unsigned maxSymbolValue,
	446	const U32* baseValue, const U8* nbAdditionalBits,
	447	unsigned tableLog, void* wksp, size_t wkspSize)
	448	{
	449	ZSTD_seqSymbol* const tableDecode = dt+1;
	450	U32 const maxSV1 = maxSymbolValue + 1;
	451	U32 const tableSize = 1 << tableLog;
	452
	453	U16* symbolNext = (U16*)wksp;
	454	BYTE* spread = (BYTE*)(symbolNext + MaxSeq + 1);
	455	U32 highThreshold = tableSize - 1;
	456
	457
	458	/* Sanity Checks */
	459	assert(maxSymbolValue <= MaxSeq);
	460	assert(tableLog <= MaxFSELog);
	461	assert(wkspSize >= ZSTD_BUILD_FSE_TABLE_WKSP_SIZE);
	462	(void)wkspSize;
	463	/* Init, lay down lowprob symbols */
	464	{ ZSTD_seqSymbol_header DTableH;
	465	DTableH.tableLog = tableLog;
	466	DTableH.fastMode = 1;
	467	{ S16 const largeLimit= (S16)(1 << (tableLog-1));
	468	U32 s;
	469	for (s=0; s<maxSV1; s++) {
	470	if (normalizedCounter[s]==-1) {
	471	tableDecode[highThreshold--].baseValue = s;
	472	symbolNext[s] = 1;
	473	} else {
	474	if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
	475	assert(normalizedCounter[s]>=0);
	476	symbolNext[s] = (U16)normalizedCounter[s];
	477	} } }
	478	ZSTD_memcpy(dt, &DTableH, sizeof(DTableH));
	479	}
	480
	481	/* Spread symbols */
	482	assert(tableSize <= 512);
	483	/* Specialized symbol spreading for the case when there are
	484	* no low probability (-1 count) symbols. When compressing
	485	* small blocks we avoid low probability symbols to hit this
	486	* case, since header decoding speed matters more.
	487	*/
	488	if (highThreshold == tableSize - 1) {
	489	size_t const tableMask = tableSize-1;
	490	size_t const step = FSE_TABLESTEP(tableSize);
	491	/* First lay down the symbols in order.
	492	* We use a uint64_t to lay down 8 bytes at a time. This reduces branch
	493	* misses since small blocks generally have small table logs, so nearly
	494	* all symbols have counts <= 8. We ensure we have 8 bytes at the end of
	495	* our buffer to handle the over-write.
	496	*/
	497	{
	498	U64 const add = 0x0101010101010101ull;
	499	size_t pos = 0;
	500	U64 sv = 0;
	501	U32 s;
	502	for (s=0; s<maxSV1; ++s, sv += add) {
	503	int i;
	504	int const n = normalizedCounter[s];
	505	MEM_write64(spread + pos, sv);
	506	for (i = 8; i < n; i += 8) {
	507	MEM_write64(spread + pos + i, sv);
	508	}
	509	pos += n;
	510	}
	511	}
	512	/* Now we spread those positions across the table.
	513	* The benefit of doing it in two stages is that we avoid the the
	514	* variable size inner loop, which caused lots of branch misses.
	515	* Now we can run through all the positions without any branch misses.
	516	* We unroll the loop twice, since that is what emperically worked best.
	517	*/
	518	{
	519	size_t position = 0;
	520	size_t s;
	521	size_t const unroll = 2;
	522	assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */
	523	for (s = 0; s < (size_t)tableSize; s += unroll) {
	524	size_t u;
	525	for (u = 0; u < unroll; ++u) {
	526	size_t const uPosition = (position + (u * step)) & tableMask;
	527	tableDecode[uPosition].baseValue = spread[s + u];
	528	}
	529	position = (position + (unroll * step)) & tableMask;
	530	}
	531	assert(position == 0);
	532	}
	533	} else {
	534	U32 const tableMask = tableSize-1;
	535	U32 const step = FSE_TABLESTEP(tableSize);
	536	U32 s, position = 0;
	537	for (s=0; s<maxSV1; s++) {
	538	int i;
	539	int const n = normalizedCounter[s];
	540	for (i=0; i<n; i++) {
	541	tableDecode[position].baseValue = s;
	542	position = (position + step) & tableMask;
	543	while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */
	544	} }
	545	assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
	546	}
	547
	548	/* Build Decoding table */
	549	{
	550	U32 u;
	551	for (u=0; u<tableSize; u++) {
	552	U32 const symbol = tableDecode[u].baseValue;
	553	U32 const nextState = symbolNext[symbol]++;
	554	tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32(nextState) );
	555	tableDecode[u].nextState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
	556	assert(nbAdditionalBits[symbol] < 255);
	557	tableDecode[u].nbAdditionalBits = nbAdditionalBits[symbol];
	558	tableDecode[u].baseValue = baseValue[symbol];
	559	}
	560	}
	561	}
	562
	563	/* Avoids the FORCE_INLINE of the _body() function. */
	564	static void ZSTD_buildFSETable_body_default(ZSTD_seqSymbol* dt,
	565	const short* normalizedCounter, unsigned maxSymbolValue,
	566	const U32* baseValue, const U8* nbAdditionalBits,
	567	unsigned tableLog, void* wksp, size_t wkspSize)
	568	{
	569	ZSTD_buildFSETable_body(dt, normalizedCounter, maxSymbolValue,
	570	baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
	571	}
	572
	573	#if DYNAMIC_BMI2
	574	BMI2_TARGET_ATTRIBUTE static void ZSTD_buildFSETable_body_bmi2(ZSTD_seqSymbol* dt,
	575	const short* normalizedCounter, unsigned maxSymbolValue,
	576	const U32* baseValue, const U8* nbAdditionalBits,
	577	unsigned tableLog, void* wksp, size_t wkspSize)
	578	{
	579	ZSTD_buildFSETable_body(dt, normalizedCounter, maxSymbolValue,
	580	baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
	581	}
	582	#endif
	583
	584	void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
	585	const short* normalizedCounter, unsigned maxSymbolValue,
	586	const U32* baseValue, const U8* nbAdditionalBits,
	587	unsigned tableLog, void* wksp, size_t wkspSize, int bmi2)
	588	{
	589	#if DYNAMIC_BMI2
	590	if (bmi2) {
	591	ZSTD_buildFSETable_body_bmi2(dt, normalizedCounter, maxSymbolValue,
	592	baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
	593	return;
	594	}
	595	#endif
	596	(void)bmi2;
	597	ZSTD_buildFSETable_body_default(dt, normalizedCounter, maxSymbolValue,
	598	baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
	599	}
	600
	601
	602	/*! ZSTD_buildSeqTable() :
	603	* @return : nb bytes read from src,
	604	* or an error code if it fails */
	605	static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymbol** DTablePtr,
	606	symbolEncodingType_e type, unsigned max, U32 maxLog,
	607	const void* src, size_t srcSize,
	608	const U32* baseValue, const U8* nbAdditionalBits,
	609	const ZSTD_seqSymbol* defaultTable, U32 flagRepeatTable,
	610	int ddictIsCold, int nbSeq, U32* wksp, size_t wkspSize,
	611	int bmi2)
	612	{
	613	switch(type)
	614	{
	615	case set_rle :
	616	RETURN_ERROR_IF(!srcSize, srcSize_wrong, "");
	617	RETURN_ERROR_IF(((const BYTE)src) > max, corruption_detected, "");
	618	{ U32 const symbol = (const BYTE)src;
	619	U32 const baseline = baseValue[symbol];
	620	U8 const nbBits = nbAdditionalBits[symbol];
	621	ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits);
	622	}
	623	*DTablePtr = DTableSpace;
	624	return 1;
	625	case set_basic :
	626	*DTablePtr = defaultTable;
	627	return 0;
	628	case set_repeat:
	629	RETURN_ERROR_IF(!flagRepeatTable, corruption_detected, "");
	630	/* prefetch FSE table if used */
	631	if (ddictIsCold && (nbSeq > 24 /* heuristic */)) {
	632	const void* const pStart = *DTablePtr;
	633	size_t const pSize = sizeof(ZSTD_seqSymbol) * (SEQSYMBOL_TABLE_SIZE(maxLog));
	634	PREFETCH_AREA(pStart, pSize);
	635	}
	636	return 0;
	637	case set_compressed :
	638	{ unsigned tableLog;
	639	S16 norm[MaxSeq+1];
	640	size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize);
	641	RETURN_ERROR_IF(FSE_isError(headerSize), corruption_detected, "");
	642	RETURN_ERROR_IF(tableLog > maxLog, corruption_detected, "");
	643	ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog, wksp, wkspSize, bmi2);
	644	*DTablePtr = DTableSpace;
	645	return headerSize;
	646	}
	647	default :
	648	assert(0);
	649	RETURN_ERROR(GENERIC, "impossible");
	650	}
	651	}
	652
	653	size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
	654	const void* src, size_t srcSize)
	655	{
	656	const BYTE* const istart = (const BYTE*)src;
	657	const BYTE* const iend = istart + srcSize;
	658	const BYTE* ip = istart;
	659	int nbSeq;
	660	DEBUGLOG(5, "ZSTD_decodeSeqHeaders");
	661
	662	/* check */
	663	RETURN_ERROR_IF(srcSize < MIN_SEQUENCES_SIZE, srcSize_wrong, "");
	664
	665	/* SeqHead */
	666	nbSeq = *ip++;
	667	if (!nbSeq) {
	668	*nbSeqPtr=0;
	669	RETURN_ERROR_IF(srcSize != 1, srcSize_wrong, "");
	670	return 1;
	671	}
	672	if (nbSeq > 0x7F) {
	673	if (nbSeq == 0xFF) {
	674	RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, "");
	675	nbSeq = MEM_readLE16(ip) + LONGNBSEQ;
	676	ip+=2;
	677	} else {
	678	RETURN_ERROR_IF(ip >= iend, srcSize_wrong, "");
	679	nbSeq = ((nbSeq-0x80)<<8) + *ip++;
	680	}
	681	}
	682	*nbSeqPtr = nbSeq;
	683
	684	/* FSE table descriptors */
	685	RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */
	686	{ symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6);
	687	symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3);
	688	symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3);
	689	ip++;
	690
	691	/* Build DTables */
	692	{ size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr,
	693	LLtype, MaxLL, LLFSELog,
	694	ip, iend-ip,
	695	LL_base, LL_bits,
	696	LL_defaultDTable, dctx->fseEntropy,
	697	dctx->ddictIsCold, nbSeq,
	698	dctx->workspace, sizeof(dctx->workspace),
	699	ZSTD_DCtx_get_bmi2(dctx));
	700	RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed");
	701	ip += llhSize;
	702	}
	703
	704	{ size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr,
	705	OFtype, MaxOff, OffFSELog,
	706	ip, iend-ip,
	707	OF_base, OF_bits,
	708	OF_defaultDTable, dctx->fseEntropy,
	709	dctx->ddictIsCold, nbSeq,
	710	dctx->workspace, sizeof(dctx->workspace),
	711	ZSTD_DCtx_get_bmi2(dctx));
	712	RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed");
	713	ip += ofhSize;
	714	}
	715
	716	{ size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr,
	717	MLtype, MaxML, MLFSELog,
	718	ip, iend-ip,
	719	ML_base, ML_bits,
	720	ML_defaultDTable, dctx->fseEntropy,
	721	dctx->ddictIsCold, nbSeq,
	722	dctx->workspace, sizeof(dctx->workspace),
	723	ZSTD_DCtx_get_bmi2(dctx));
	724	RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed");
	725	ip += mlhSize;
	726	}
	727	}
	728
	729	return ip-istart;
	730	}
	731
	732
	733	typedef struct {
	734	size_t litLength;
	735	size_t matchLength;
	736	size_t offset;
	737	} seq_t;
	738
	739	typedef struct {
	740	size_t state;
	741	const ZSTD_seqSymbol* table;
	742	} ZSTD_fseState;
	743
	744	typedef struct {
	745	BIT_DStream_t DStream;
	746	ZSTD_fseState stateLL;
	747	ZSTD_fseState stateOffb;
	748	ZSTD_fseState stateML;
	749	size_t prevOffset[ZSTD_REP_NUM];
	750	} seqState_t;
	751
	752	/*! ZSTD_overlapCopy8() :
	753	* Copies 8 bytes from ip to op and updates op and ip where ip <= op.
	754	* If the offset is < 8 then the offset is spread to at least 8 bytes.
	755	*
	756	* Precondition: ip <= op
	757	* Postcondition: op - op >= 8
	758	*/
	759	HINT_INLINE void ZSTD_overlapCopy8(BYTE op, BYTE const ip, size_t offset) {
	760	assert(ip <= op);
	761	if (offset < 8) {
	762	/* close range match, overlap */
	763	static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
	764	static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
	765	int const sub2 = dec64table[offset];
	766	(op)[0] = (ip)[0];
	767	(op)[1] = (ip)[1];
	768	(op)[2] = (ip)[2];
	769	(op)[3] = (ip)[3];
	770	*ip += dec32table[offset];
	771	ZSTD_copy4(op+4, ip);
	772	*ip -= sub2;
	773	} else {
	774	ZSTD_copy8(op, ip);
	775	}
	776	*ip += 8;
	777	*op += 8;
	778	assert(op - ip >= 8);
	779	}
	780
	781	/*! ZSTD_safecopy() :
	782	* Specialized version of memcpy() that is allowed to READ up to WILDCOPY_OVERLENGTH past the input buffer
	783	* and write up to 16 bytes past oend_w (op >= oend_w is allowed).
	784	* This function is only called in the uncommon case where the sequence is near the end of the block. It
	785	* should be fast for a single long sequence, but can be slow for several short sequences.
	786	*
	787	* @param ovtype controls the overlap detection
	788	* - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart.
	789	* - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart.
	790	* The src buffer must be before the dst buffer.
	791	*/
	792	static void ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) {
	793	ptrdiff_t const diff = op - ip;
	794	BYTE* const oend = op + length;
	795
	796	assert((ovtype == ZSTD_no_overlap && (diff <= -8 \|\| diff >= 8 \|\| op >= oend_w)) \|\|
	797	(ovtype == ZSTD_overlap_src_before_dst && diff >= 0));
	798
	799	if (length < 8) {
	800	/* Handle short lengths. */
	801	while (op < oend) op++ = ip++;
	802	return;
	803	}
	804	if (ovtype == ZSTD_overlap_src_before_dst) {
	805	/* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */
	806	assert(length >= 8);
	807	ZSTD_overlapCopy8(&op, &ip, diff);
	808	length -= 8;
	809	assert(op - ip >= 8);
	810	assert(op <= oend);
	811	}
	812
	813	if (oend <= oend_w) {
	814	/* No risk of overwrite. */
	815	ZSTD_wildcopy(op, ip, length, ovtype);
	816	return;
	817	}
	818	if (op <= oend_w) {
	819	/* Wildcopy until we get close to the end. */
	820	assert(oend > oend_w);
	821	ZSTD_wildcopy(op, ip, oend_w - op, ovtype);
	822	ip += oend_w - op;
	823	op += oend_w - op;
	824	}
	825	/* Handle the leftovers. */
	826	while (op < oend) op++ = ip++;
	827	}
	828
	829	/* ZSTD_safecopyDstBeforeSrc():
	830	* This version allows overlap with dst before src, or handles the non-overlap case with dst after src
	831	* Kept separate from more common ZSTD_safecopy case to avoid performance impact to the safecopy common case */
	832	static void ZSTD_safecopyDstBeforeSrc(BYTE* op, BYTE const* ip, ptrdiff_t length) {
	833	ptrdiff_t const diff = op - ip;
	834	BYTE* const oend = op + length;
	835
	836	if (length < 8 \|\| diff > -8) {
	837	/* Handle short lengths, close overlaps, and dst not before src. */
	838	while (op < oend) op++ = ip++;
	839	return;
	840	}
	841
	842	if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) {
	843	ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap);
	844	ip += oend - WILDCOPY_OVERLENGTH - op;
	845	op += oend - WILDCOPY_OVERLENGTH - op;
	846	}
	847
	848	/* Handle the leftovers. */
	849	while (op < oend) op++ = ip++;
	850	}
	851
	852	/* ZSTD_execSequenceEnd():
	853	* This version handles cases that are near the end of the output buffer. It requires
	854	* more careful checks to make sure there is no overflow. By separating out these hard
	855	* and unlikely cases, we can speed up the common cases.
	856	*
	857	* NOTE: This function needs to be fast for a single long sequence, but doesn't need
	858	* to be optimized for many small sequences, since those fall into ZSTD_execSequence().
	859	*/
	860	FORCE_NOINLINE
	861	size_t ZSTD_execSequenceEnd(BYTE* op,
	862	BYTE* const oend, seq_t sequence,
	863	const BYTE** litPtr, const BYTE* const litLimit,
	864	const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
	865	{
	866	BYTE* const oLitEnd = op + sequence.litLength;
	867	size_t const sequenceLength = sequence.litLength + sequence.matchLength;
	868	const BYTE* const iLitEnd = *litPtr + sequence.litLength;
	869	const BYTE* match = oLitEnd - sequence.offset;
	870	BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH;
	871
	872	/* bounds checks : careful of address space overflow in 32-bit mode */
	873	RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer");
	874	RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer");
	875	assert(op < op + sequenceLength);
	876	assert(oLitEnd < op + sequenceLength);
	877
	878	/* copy literals */
	879	ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap);
	880	op = oLitEnd;
	881	*litPtr = iLitEnd;
	882
	883	/* copy Match */
	884	if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
	885	/* offset beyond prefix */
	886	RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, "");
	887	match = dictEnd - (prefixStart - match);
	888	if (match + sequence.matchLength <= dictEnd) {
	889	ZSTD_memmove(oLitEnd, match, sequence.matchLength);
	890	return sequenceLength;
	891	}
	892	/* span extDict & currentPrefixSegment */
	893	{ size_t const length1 = dictEnd - match;
	894	ZSTD_memmove(oLitEnd, match, length1);
	895	op = oLitEnd + length1;
	896	sequence.matchLength -= length1;
	897	match = prefixStart;
	898	}
	899	}
	900	ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst);
	901	return sequenceLength;
	902	}
	903
	904	/* ZSTD_execSequenceEndSplitLitBuffer():
	905	* This version is intended to be used during instances where the litBuffer is still split. It is kept separate to avoid performance impact for the good case.
	906	*/
	907	FORCE_NOINLINE
	908	size_t ZSTD_execSequenceEndSplitLitBuffer(BYTE* op,
	909	BYTE* const oend, const BYTE* const oend_w, seq_t sequence,
	910	const BYTE** litPtr, const BYTE* const litLimit,
	911	const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
	912	{
	913	BYTE* const oLitEnd = op + sequence.litLength;
	914	size_t const sequenceLength = sequence.litLength + sequence.matchLength;
	915	const BYTE* const iLitEnd = *litPtr + sequence.litLength;
	916	const BYTE* match = oLitEnd - sequence.offset;
	917
	918
	919	/* bounds checks : careful of address space overflow in 32-bit mode */
	920	RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer");
	921	RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer");
	922	assert(op < op + sequenceLength);
	923	assert(oLitEnd < op + sequenceLength);
	924
	925	/* copy literals */
	926	RETURN_ERROR_IF(op > litPtr && op < litPtr + sequence.litLength, dstSize_tooSmall, "output should not catch up to and overwrite literal buffer");
	927	ZSTD_safecopyDstBeforeSrc(op, *litPtr, sequence.litLength);
	928	op = oLitEnd;
	929	*litPtr = iLitEnd;
	930
	931	/* copy Match */
	932	if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
	933	/* offset beyond prefix */
	934	RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, "");
	935	match = dictEnd - (prefixStart - match);
	936	if (match + sequence.matchLength <= dictEnd) {
	937	ZSTD_memmove(oLitEnd, match, sequence.matchLength);
	938	return sequenceLength;
	939	}
	940	/* span extDict & currentPrefixSegment */
	941	{ size_t const length1 = dictEnd - match;
	942	ZSTD_memmove(oLitEnd, match, length1);
	943	op = oLitEnd + length1;
	944	sequence.matchLength -= length1;
	945	match = prefixStart;
	946	}
	947	}
	948	ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst);
	949	return sequenceLength;
	950	}
	951
	952	HINT_INLINE
	953	size_t ZSTD_execSequence(BYTE* op,
	954	BYTE* const oend, seq_t sequence,
	955	const BYTE** litPtr, const BYTE* const litLimit,
	956	const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
	957	{
	958	BYTE* const oLitEnd = op + sequence.litLength;
	959	size_t const sequenceLength = sequence.litLength + sequence.matchLength;
	960	BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
	961	BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* risk : address space underflow on oend=NULL */
	962	const BYTE* const iLitEnd = *litPtr + sequence.litLength;
	963	const BYTE* match = oLitEnd - sequence.offset;
	964
	965	assert(op != NULL /* Precondition */);
	966	assert(oend_w < oend /* No underflow */);
	967	/* Handle edge cases in a slow path:
	968	* - Read beyond end of literals
	969	* - Match end is within WILDCOPY_OVERLIMIT of oend
	970	* - 32-bit mode and the match length overflows
	971	*/
	972	if (UNLIKELY(
	973	iLitEnd > litLimit \|\|
	974	oMatchEnd > oend_w \|\|
	975	(MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH)))
	976	return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd);
	977
	978	/* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */
	979	assert(op <= oLitEnd /* No overflow */);
	980	assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */);
	981	assert(oMatchEnd <= oend /* No underflow */);
	982	assert(iLitEnd <= litLimit /* Literal length is in bounds */);
	983	assert(oLitEnd <= oend_w /* Can wildcopy literals */);
	984	assert(oMatchEnd <= oend_w /* Can wildcopy matches */);
	985
	986	/* Copy Literals:
	987	* Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9.
	988	* We likely don't need the full 32-byte wildcopy.
	989	*/
	990	assert(WILDCOPY_OVERLENGTH >= 16);
	991	ZSTD_copy16(op, (*litPtr));
	992	if (UNLIKELY(sequence.litLength > 16)) {
	993	ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap);
	994	}
	995	op = oLitEnd;
	996	litPtr = iLitEnd; / update for next sequence */
	997
	998	/* Copy Match */
	999	if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
	1000	/* offset beyond prefix -> go into extDict */
	1001	RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, "");
	1002	match = dictEnd + (match - prefixStart);
	1003	if (match + sequence.matchLength <= dictEnd) {
	1004	ZSTD_memmove(oLitEnd, match, sequence.matchLength);
	1005	return sequenceLength;
	1006	}
	1007	/* span extDict & currentPrefixSegment */
	1008	{ size_t const length1 = dictEnd - match;
	1009	ZSTD_memmove(oLitEnd, match, length1);
	1010	op = oLitEnd + length1;
	1011	sequence.matchLength -= length1;
	1012	match = prefixStart;
	1013	}
	1014	}
	1015	/* Match within prefix of 1 or more bytes */
	1016	assert(op <= oMatchEnd);
	1017	assert(oMatchEnd <= oend_w);
	1018	assert(match >= prefixStart);
	1019	assert(sequence.matchLength >= 1);
	1020
	1021	/* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy
	1022	* without overlap checking.
	1023	*/
	1024	if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) {
	1025	/* We bet on a full wildcopy for matches, since we expect matches to be
	1026	* longer than literals (in general). In silesia, ~10% of matches are longer
	1027	* than 16 bytes.
	1028	*/
	1029	ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap);
	1030	return sequenceLength;
	1031	}
	1032	assert(sequence.offset < WILDCOPY_VECLEN);
	1033
	1034	/* Copy 8 bytes and spread the offset to be >= 8. */
	1035	ZSTD_overlapCopy8(&op, &match, sequence.offset);
	1036
	1037	/* If the match length is > 8 bytes, then continue with the wildcopy. */
	1038	if (sequence.matchLength > 8) {
	1039	assert(op < oMatchEnd);
	1040	ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst);
	1041	}
	1042	return sequenceLength;
	1043	}
	1044
	1045	HINT_INLINE
	1046	size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op,
	1047	BYTE* const oend, const BYTE* const oend_w, seq_t sequence,
	1048	const BYTE** litPtr, const BYTE* const litLimit,
	1049	const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
	1050	{
	1051	BYTE* const oLitEnd = op + sequence.litLength;
	1052	size_t const sequenceLength = sequence.litLength + sequence.matchLength;
	1053	BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
	1054	const BYTE* const iLitEnd = *litPtr + sequence.litLength;
	1055	const BYTE* match = oLitEnd - sequence.offset;
	1056
	1057	assert(op != NULL /* Precondition */);
	1058	assert(oend_w < oend /* No underflow */);
	1059	/* Handle edge cases in a slow path:
	1060	* - Read beyond end of literals
	1061	* - Match end is within WILDCOPY_OVERLIMIT of oend
	1062	* - 32-bit mode and the match length overflows
	1063	*/
	1064	if (UNLIKELY(
	1065	iLitEnd > litLimit \|\|
	1066	oMatchEnd > oend_w \|\|
	1067	(MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH)))
	1068	return ZSTD_execSequenceEndSplitLitBuffer(op, oend, oend_w, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd);
	1069
	1070	/* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */
	1071	assert(op <= oLitEnd /* No overflow */);
	1072	assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */);
	1073	assert(oMatchEnd <= oend /* No underflow */);
	1074	assert(iLitEnd <= litLimit /* Literal length is in bounds */);
	1075	assert(oLitEnd <= oend_w /* Can wildcopy literals */);
	1076	assert(oMatchEnd <= oend_w /* Can wildcopy matches */);
	1077
	1078	/* Copy Literals:
	1079	* Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9.
	1080	* We likely don't need the full 32-byte wildcopy.
	1081	*/
	1082	assert(WILDCOPY_OVERLENGTH >= 16);
	1083	ZSTD_copy16(op, (*litPtr));
	1084	if (UNLIKELY(sequence.litLength > 16)) {
	1085	ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap);
	1086	}
	1087	op = oLitEnd;
	1088	litPtr = iLitEnd; / update for next sequence */
	1089
	1090	/* Copy Match */
	1091	if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
	1092	/* offset beyond prefix -> go into extDict */
	1093	RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, "");
	1094	match = dictEnd + (match - prefixStart);
	1095	if (match + sequence.matchLength <= dictEnd) {
	1096	ZSTD_memmove(oLitEnd, match, sequence.matchLength);
	1097	return sequenceLength;
	1098	}
	1099	/* span extDict & currentPrefixSegment */
	1100	{ size_t const length1 = dictEnd - match;
	1101	ZSTD_memmove(oLitEnd, match, length1);
	1102	op = oLitEnd + length1;
	1103	sequence.matchLength -= length1;
	1104	match = prefixStart;
	1105	} }
	1106	/* Match within prefix of 1 or more bytes */
	1107	assert(op <= oMatchEnd);
	1108	assert(oMatchEnd <= oend_w);
	1109	assert(match >= prefixStart);
	1110	assert(sequence.matchLength >= 1);
	1111
	1112	/* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy
	1113	* without overlap checking.
	1114	*/
	1115	if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) {
	1116	/* We bet on a full wildcopy for matches, since we expect matches to be
	1117	* longer than literals (in general). In silesia, ~10% of matches are longer
	1118	* than 16 bytes.
	1119	*/
	1120	ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap);
	1121	return sequenceLength;
	1122	}
	1123	assert(sequence.offset < WILDCOPY_VECLEN);
	1124
	1125	/* Copy 8 bytes and spread the offset to be >= 8. */
	1126	ZSTD_overlapCopy8(&op, &match, sequence.offset);
	1127
	1128	/* If the match length is > 8 bytes, then continue with the wildcopy. */
	1129	if (sequence.matchLength > 8) {
	1130	assert(op < oMatchEnd);
	1131	ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst);
	1132	}
	1133	return sequenceLength;
	1134	}
	1135
	1136
	1137	static void
	1138	ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt)
	1139	{
	1140	const void* ptr = dt;
	1141	const ZSTD_seqSymbol_header* const DTableH = (const ZSTD_seqSymbol_header*)ptr;
	1142	DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
	1143	DEBUGLOG(6, "ZSTD_initFseState : val=%u using %u bits",
	1144	(U32)DStatePtr->state, DTableH->tableLog);
	1145	BIT_reloadDStream(bitD);
	1146	DStatePtr->table = dt + 1;
	1147	}
	1148
	1149	FORCE_INLINE_TEMPLATE void
	1150	ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 nextState, U32 nbBits)
	1151	{
	1152	size_t const lowBits = BIT_readBits(bitD, nbBits);
	1153	DStatePtr->state = nextState + lowBits;
	1154	}
	1155
	1156	/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum
	1157	* offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1)
	1158	* bits before reloading. This value is the maximum number of bytes we read
	1159	* after reloading when we are decoding long offsets.
	1160	*/
	1161	#define LONG_OFFSETS_MAX_EXTRA_BITS_32 \
	1162	(ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 \
	1163	? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \
	1164	: 0)
	1165
	1166	typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e;
	1167
	1168	FORCE_INLINE_TEMPLATE seq_t
	1169	ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets)
	1170	{
	1171	seq_t seq;
	1172	const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state;
	1173	const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state;
	1174	const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state;
	1175	seq.matchLength = mlDInfo->baseValue;
	1176	seq.litLength = llDInfo->baseValue;
	1177	{ U32 const ofBase = ofDInfo->baseValue;
	1178	BYTE const llBits = llDInfo->nbAdditionalBits;
	1179	BYTE const mlBits = mlDInfo->nbAdditionalBits;
	1180	BYTE const ofBits = ofDInfo->nbAdditionalBits;
	1181	BYTE const totalBits = llBits+mlBits+ofBits;
	1182
	1183	U16 const llNext = llDInfo->nextState;
	1184	U16 const mlNext = mlDInfo->nextState;
	1185	U16 const ofNext = ofDInfo->nextState;
	1186	U32 const llnbBits = llDInfo->nbBits;
	1187	U32 const mlnbBits = mlDInfo->nbBits;
	1188	U32 const ofnbBits = ofDInfo->nbBits;
	1189	/*
	1190	* As gcc has better branch and block analyzers, sometimes it is only
	1191	* valuable to mark likelyness for clang, it gives around 3-4% of
	1192	* performance.
	1193	*/
	1194
	1195	/* sequence */
	1196	{ size_t offset;
	1197	#if defined(__clang__)
	1198	if (LIKELY(ofBits > 1)) {
	1199	#else
	1200	if (ofBits > 1) {
	1201	#endif
	1202	ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
	1203	ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
	1204	assert(ofBits <= MaxOff);
	1205	if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) {
	1206	U32 const extraBits = ofBits - MIN(ofBits, 32 - seqState->DStream.bitsConsumed);
	1207	offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
	1208	BIT_reloadDStream(&seqState->DStream);
	1209	if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits);
	1210	assert(extraBits <= LONG_OFFSETS_MAX_EXTRA_BITS_32); /* to avoid another reload */
	1211	} else {
	1212	offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/>0/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */
	1213	if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);
	1214	}
	1215	seqState->prevOffset[2] = seqState->prevOffset[1];
	1216	seqState->prevOffset[1] = seqState->prevOffset[0];
	1217	seqState->prevOffset[0] = offset;
	1218	} else {
	1219	U32 const ll0 = (llDInfo->baseValue == 0);
	1220	if (LIKELY((ofBits == 0))) {
	1221	offset = seqState->prevOffset[ll0];
	1222	seqState->prevOffset[1] = seqState->prevOffset[!ll0];
	1223	seqState->prevOffset[0] = offset;
	1224	} else {
	1225	offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1);
	1226	{ size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
	1227	temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */
	1228	if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1];
	1229	seqState->prevOffset[1] = seqState->prevOffset[0];
	1230	seqState->prevOffset[0] = offset = temp;
	1231	} } }
	1232	seq.offset = offset;
	1233	}
	1234
	1235	#if defined(__clang__)
	1236	if (UNLIKELY(mlBits > 0))
	1237	#else
	1238	if (mlBits > 0)
	1239	#endif
	1240	seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/>0/);
	1241
	1242	if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32))
	1243	BIT_reloadDStream(&seqState->DStream);
	1244	if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog)))
	1245	BIT_reloadDStream(&seqState->DStream);
	1246	/* Ensure there are enough bits to read the rest of data in 64-bit mode. */
	1247	ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64);
	1248
	1249	#if defined(__clang__)
	1250	if (UNLIKELY(llBits > 0))
	1251	#else
	1252	if (llBits > 0)
	1253	#endif
	1254	seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/>0/);
	1255
	1256	if (MEM_32bits())
	1257	BIT_reloadDStream(&seqState->DStream);
	1258
	1259	DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u",
	1260	(U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
	1261
	1262	ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */
	1263	ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */
	1264	if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */
	1265	ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */
	1266	}
	1267
	1268	return seq;
	1269	}
	1270
	1271	#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
	1272	MEM_STATIC int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd)
	1273	{
	1274	size_t const windowSize = dctx->fParams.windowSize;
	1275	/* No dictionary used. */
	1276	if (dctx->dictContentEndForFuzzing == NULL) return 0;
	1277	/* Dictionary is our prefix. */
	1278	if (prefixStart == dctx->dictContentBeginForFuzzing) return 1;
	1279	/* Dictionary is not our ext-dict. */
	1280	if (dctx->dictEnd != dctx->dictContentEndForFuzzing) return 0;
	1281	/* Dictionary is not within our window size. */
	1282	if ((size_t)(oLitEnd - prefixStart) >= windowSize) return 0;
	1283	/* Dictionary is active. */
	1284	return 1;
	1285	}
	1286
	1287	MEM_STATIC void ZSTD_assertValidSequence(
	1288	ZSTD_DCtx const* dctx,
	1289	BYTE const* op, BYTE const* oend,
	1290	seq_t const seq,
	1291	BYTE const* prefixStart, BYTE const* virtualStart)
	1292	{
	1293	#if DEBUGLEVEL >= 1
	1294	size_t const windowSize = dctx->fParams.windowSize;
	1295	size_t const sequenceSize = seq.litLength + seq.matchLength;
	1296	BYTE const* const oLitEnd = op + seq.litLength;
	1297	DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u",
	1298	(U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
	1299	assert(op <= oend);
	1300	assert((size_t)(oend - op) >= sequenceSize);
	1301	assert(sequenceSize <= ZSTD_BLOCKSIZE_MAX);
	1302	if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) {
	1303	size_t const dictSize = (size_t)((char const)dctx->dictContentEndForFuzzing - (char const)dctx->dictContentBeginForFuzzing);
	1304	/* Offset must be within the dictionary. */
	1305	assert(seq.offset <= (size_t)(oLitEnd - virtualStart));
	1306	assert(seq.offset <= windowSize + dictSize);
	1307	} else {
	1308	/* Offset must be within our window. */
	1309	assert(seq.offset <= windowSize);
	1310	}
	1311	#else
	1312	(void)dctx, (void)op, (void)oend, (void)seq, (void)prefixStart, (void)virtualStart;
	1313	#endif
	1314	}
	1315	#endif
	1316
	1317	#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
	1318
	1319
	1320	FORCE_INLINE_TEMPLATE size_t
	1321	DONT_VECTORIZE
	1322	ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx,
	1323	void* dst, size_t maxDstSize,
	1324	const void* seqStart, size_t seqSize, int nbSeq,
	1325	const ZSTD_longOffset_e isLongOffset,
	1326	const int frame)
	1327	{
	1328	const BYTE* ip = (const BYTE*)seqStart;
	1329	const BYTE* const iend = ip + seqSize;
	1330	BYTE* const ostart = (BYTE*)dst;
	1331	BYTE* const oend = ostart + maxDstSize;
	1332	BYTE* op = ostart;
	1333	const BYTE* litPtr = dctx->litPtr;
	1334	const BYTE* litBufferEnd = dctx->litBufferEnd;
	1335	const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart);
	1336	const BYTE* const vBase = (const BYTE*) (dctx->virtualStart);
	1337	const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
	1338	DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer");
	1339	(void)frame;
	1340
	1341	/* Regen sequences */
	1342	if (nbSeq) {
	1343	seqState_t seqState;
	1344	dctx->fseEntropy = 1;
	1345	{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
	1346	RETURN_ERROR_IF(
	1347	ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)),
	1348	corruption_detected, "");
	1349	ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
	1350	ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
	1351	ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
	1352	assert(dst != NULL);
	1353
	1354	ZSTD_STATIC_ASSERT(
	1355	BIT_DStream_unfinished < BIT_DStream_completed &&
	1356	BIT_DStream_endOfBuffer < BIT_DStream_completed &&
	1357	BIT_DStream_completed < BIT_DStream_overflow);
	1358
	1359	/* decompress without overrunning litPtr begins */
	1360	{
	1361	seq_t sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
	1362	/* Align the decompression loop to 32 + 16 bytes.
	1363	*
	1364	* zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression
	1365	* speed swings based on the alignment of the decompression loop. This
	1366	* performance swing is caused by parts of the decompression loop falling
	1367	* out of the DSB. The entire decompression loop should fit in the DSB,
	1368	* when it can't we get much worse performance. You can measure if you've
	1369	* hit the good case or the bad case with this perf command for some
	1370	* compressed file test.zst:
	1371	*
	1372	* perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \
	1373	* -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst
	1374	*
	1375	* If you see most cycles served out of the MITE you've hit the bad case.
	1376	* If you see most cycles served out of the DSB you've hit the good case.
	1377	* If it is pretty even then you may be in an okay case.
	1378	*
	1379	* This issue has been reproduced on the following CPUs:
	1380	* - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9
	1381	* Use Instruments->Counters to get DSB/MITE cycles.
	1382	* I never got performance swings, but I was able to
	1383	* go from the good case of mostly DSB to half of the
	1384	* cycles served from MITE.
	1385	* - Coffeelake: Intel i9-9900k
	1386	* - Coffeelake: Intel i7-9700k
	1387	*
	1388	* I haven't been able to reproduce the instability or DSB misses on any
	1389	* of the following CPUS:
	1390	* - Haswell
	1391	* - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH
	1392	* - Skylake
	1393	*
	1394	* Alignment is done for each of the three major decompression loops:
	1395	* - ZSTD_decompressSequences_bodySplitLitBuffer - presplit section of the literal buffer
	1396	* - ZSTD_decompressSequences_bodySplitLitBuffer - postsplit section of the literal buffer
	1397	* - ZSTD_decompressSequences_body
	1398	* Alignment choices are made to minimize large swings on bad cases and influence on performance
	1399	* from changes external to this code, rather than to overoptimize on the current commit.
	1400	*
	1401	* If you are seeing performance stability this script can help test.
	1402	* It tests on 4 commits in zstd where I saw performance change.
	1403	*
	1404	* https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4
	1405	*/
	1406	#if defined(__GNUC__) && defined(__x86_64__)
	1407	__asm__(".p2align 6");
	1408	# if __GNUC__ >= 7
	1409	/* good for gcc-7, gcc-9, and gcc-11 */
	1410	__asm__("nop");
	1411	__asm__(".p2align 5");
	1412	__asm__("nop");
	1413	__asm__(".p2align 4");
	1414	# if __GNUC__ == 8 \|\| __GNUC__ == 10
	1415	/* good for gcc-8 and gcc-10 */
	1416	__asm__("nop");
	1417	__asm__(".p2align 3");
	1418	# endif
	1419	# endif
	1420	#endif
	1421
	1422	/* Handle the initial state where litBuffer is currently split between dst and litExtraBuffer */
	1423	for (; litPtr + sequence.litLength <= dctx->litBufferEnd; ) {
	1424	size_t const oneSeqSize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
	1425	#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
	1426	assert(!ZSTD_isError(oneSeqSize));
	1427	if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
	1428	#endif
	1429	if (UNLIKELY(ZSTD_isError(oneSeqSize)))
	1430	return oneSeqSize;
	1431	DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
	1432	op += oneSeqSize;
	1433	if (UNLIKELY(!--nbSeq))
	1434	break;
	1435	BIT_reloadDStream(&(seqState.DStream));
	1436	sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
	1437	}
	1438
	1439	/* If there are more sequences, they will need to read literals from litExtraBuffer; copy over the remainder from dst and update litPtr and litEnd */
	1440	if (nbSeq > 0) {
	1441	const size_t leftoverLit = dctx->litBufferEnd - litPtr;
	1442	if (leftoverLit)
	1443	{
	1444	RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
	1445	ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
	1446	sequence.litLength -= leftoverLit;
	1447	op += leftoverLit;
	1448	}
	1449	litPtr = dctx->litExtraBuffer;
	1450	litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
	1451	dctx->litBufferLocation = ZSTD_not_in_dst;
	1452	{
	1453	size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
	1454	#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
	1455	assert(!ZSTD_isError(oneSeqSize));
	1456	if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
	1457	#endif
	1458	if (UNLIKELY(ZSTD_isError(oneSeqSize)))
	1459	return oneSeqSize;
	1460	DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
	1461	op += oneSeqSize;
	1462	if (--nbSeq)
	1463	BIT_reloadDStream(&(seqState.DStream));
	1464	}
	1465	}
	1466	}
	1467
	1468	if (nbSeq > 0) /* there is remaining lit from extra buffer */
	1469	{
	1470
	1471	#if defined(__GNUC__) && defined(__x86_64__)
	1472	__asm__(".p2align 6");
	1473	__asm__("nop");
	1474	# if __GNUC__ != 7
	1475	/* worse for gcc-7 better for gcc-8, gcc-9, and gcc-10 and clang */
	1476	__asm__(".p2align 4");
	1477	__asm__("nop");
	1478	__asm__(".p2align 3");
	1479	# elif __GNUC__ >= 11
	1480	__asm__(".p2align 3");
	1481	# else
	1482	__asm__(".p2align 5");
	1483	__asm__("nop");
	1484	__asm__(".p2align 3");
	1485	# endif
	1486	#endif
	1487
	1488	for (; ; ) {
	1489	seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
	1490	size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
	1491	#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
	1492	assert(!ZSTD_isError(oneSeqSize));
	1493	if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
	1494	#endif
	1495	if (UNLIKELY(ZSTD_isError(oneSeqSize)))
	1496	return oneSeqSize;
	1497	DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
	1498	op += oneSeqSize;
	1499	if (UNLIKELY(!--nbSeq))
	1500	break;
	1501	BIT_reloadDStream(&(seqState.DStream));
	1502	}
	1503	}
	1504
	1505	/* check if reached exact end */
	1506	DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer: after decode loop, remaining nbSeq : %i", nbSeq);
	1507	RETURN_ERROR_IF(nbSeq, corruption_detected, "");
	1508	RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, "");
	1509	/* save reps for next block */
	1510	{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
	1511	}
	1512
	1513	/* last literal segment */
	1514	if (dctx->litBufferLocation == ZSTD_split) /* split hasn't been reached yet, first get dst then copy litExtraBuffer */
	1515	{
	1516	size_t const lastLLSize = litBufferEnd - litPtr;
	1517	RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, "");
	1518	if (op != NULL) {
	1519	ZSTD_memmove(op, litPtr, lastLLSize);
	1520	op += lastLLSize;
	1521	}
	1522	litPtr = dctx->litExtraBuffer;
	1523	litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
	1524	dctx->litBufferLocation = ZSTD_not_in_dst;
	1525	}
	1526	{ size_t const lastLLSize = litBufferEnd - litPtr;
	1527	RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
	1528	if (op != NULL) {
	1529	ZSTD_memcpy(op, litPtr, lastLLSize);
	1530	op += lastLLSize;
	1531	}
	1532	}
	1533
	1534	return op-ostart;
	1535	}
	1536
	1537	FORCE_INLINE_TEMPLATE size_t
	1538	DONT_VECTORIZE
	1539	ZSTD_decompressSequences_body(ZSTD_DCtx* dctx,
	1540	void* dst, size_t maxDstSize,
	1541	const void* seqStart, size_t seqSize, int nbSeq,
	1542	const ZSTD_longOffset_e isLongOffset,
	1543	const int frame)
	1544	{
	1545	const BYTE* ip = (const BYTE*)seqStart;
	1546	const BYTE* const iend = ip + seqSize;
	1547	BYTE* const ostart = (BYTE*)dst;
	1548	BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ostart + maxDstSize : dctx->litBuffer;
	1549	BYTE* op = ostart;
	1550	const BYTE* litPtr = dctx->litPtr;
	1551	const BYTE* const litEnd = litPtr + dctx->litSize;
	1552	const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart);
	1553	const BYTE* const vBase = (const BYTE*)(dctx->virtualStart);
	1554	const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd);
	1555	DEBUGLOG(5, "ZSTD_decompressSequences_body");
	1556	(void)frame;
	1557
	1558	/* Regen sequences */
	1559	if (nbSeq) {
	1560	seqState_t seqState;
	1561	dctx->fseEntropy = 1;
	1562	{ U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
	1563	RETURN_ERROR_IF(
	1564	ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)),
	1565	corruption_detected, "");
	1566	ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
	1567	ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
	1568	ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
	1569	assert(dst != NULL);
	1570
	1571	ZSTD_STATIC_ASSERT(
	1572	BIT_DStream_unfinished < BIT_DStream_completed &&
	1573	BIT_DStream_endOfBuffer < BIT_DStream_completed &&
	1574	BIT_DStream_completed < BIT_DStream_overflow);
	1575
	1576	#if defined(__GNUC__) && defined(__x86_64__)
	1577	__asm__(".p2align 6");
	1578	__asm__("nop");
	1579	# if __GNUC__ >= 7
	1580	__asm__(".p2align 5");
	1581	__asm__("nop");
	1582	__asm__(".p2align 3");
	1583	# else
	1584	__asm__(".p2align 4");
	1585	__asm__("nop");
	1586	__asm__(".p2align 3");
	1587	# endif
	1588	#endif
	1589
	1590	for ( ; ; ) {
	1591	seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
	1592	size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd);
	1593	#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
	1594	assert(!ZSTD_isError(oneSeqSize));
	1595	if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
	1596	#endif
	1597	if (UNLIKELY(ZSTD_isError(oneSeqSize)))
	1598	return oneSeqSize;
	1599	DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
	1600	op += oneSeqSize;
	1601	if (UNLIKELY(!--nbSeq))
	1602	break;
	1603	BIT_reloadDStream(&(seqState.DStream));
	1604	}
	1605
	1606	/* check if reached exact end */
	1607	DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq);
	1608	RETURN_ERROR_IF(nbSeq, corruption_detected, "");
	1609	RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, "");
	1610	/* save reps for next block */
	1611	{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
	1612	}
	1613
	1614	/* last literal segment */
	1615	{ size_t const lastLLSize = litEnd - litPtr;
	1616	RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
	1617	if (op != NULL) {
	1618	ZSTD_memcpy(op, litPtr, lastLLSize);
	1619	op += lastLLSize;
	1620	}
	1621	}
	1622
	1623	return op-ostart;
	1624	}
	1625
	1626	static size_t
	1627	ZSTD_decompressSequences_default(ZSTD_DCtx* dctx,
	1628	void* dst, size_t maxDstSize,
	1629	const void* seqStart, size_t seqSize, int nbSeq,
	1630	const ZSTD_longOffset_e isLongOffset,
	1631	const int frame)
	1632	{
	1633	return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
	1634	}
	1635
	1636	static size_t
	1637	ZSTD_decompressSequencesSplitLitBuffer_default(ZSTD_DCtx* dctx,
	1638	void* dst, size_t maxDstSize,
	1639	const void* seqStart, size_t seqSize, int nbSeq,
	1640	const ZSTD_longOffset_e isLongOffset,
	1641	const int frame)
	1642	{
	1643	return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
	1644	}
	1645	#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
	1646
	1647	#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
	1648
	1649	FORCE_INLINE_TEMPLATE size_t
	1650	ZSTD_prefetchMatch(size_t prefetchPos, seq_t const sequence,
	1651	const BYTE* const prefixStart, const BYTE* const dictEnd)
	1652	{
	1653	prefetchPos += sequence.litLength;
	1654	{ const BYTE* const matchBase = (sequence.offset > prefetchPos) ? dictEnd : prefixStart;
	1655	const BYTE* const match = matchBase + prefetchPos - sequence.offset; /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted.
	1656	* No consequence though : memory address is only used for prefetching, not for dereferencing */
	1657	PREFETCH_L1(match); PREFETCH_L1(match+CACHELINE_SIZE); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */
	1658	}
	1659	return prefetchPos + sequence.matchLength;
	1660	}
	1661
	1662	/* This decoding function employs prefetching
	1663	* to reduce latency impact of cache misses.
	1664	* It's generally employed when block contains a significant portion of long-distance matches
	1665	* or when coupled with a "cold" dictionary */
	1666	FORCE_INLINE_TEMPLATE size_t
	1667	ZSTD_decompressSequencesLong_body(
	1668	ZSTD_DCtx* dctx,
	1669	void* dst, size_t maxDstSize,
	1670	const void* seqStart, size_t seqSize, int nbSeq,
	1671	const ZSTD_longOffset_e isLongOffset,
	1672	const int frame)
	1673	{
	1674	const BYTE* ip = (const BYTE*)seqStart;
	1675	const BYTE* const iend = ip + seqSize;
	1676	BYTE* const ostart = (BYTE*)dst;
	1677	BYTE* const oend = dctx->litBufferLocation == ZSTD_in_dst ? dctx->litBuffer : ostart + maxDstSize;
	1678	BYTE* op = ostart;
	1679	const BYTE* litPtr = dctx->litPtr;
	1680	const BYTE* litBufferEnd = dctx->litBufferEnd;
	1681	const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart);
	1682	const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart);
	1683	const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
	1684	(void)frame;
	1685
	1686	/* Regen sequences */
	1687	if (nbSeq) {
	1688	#define STORED_SEQS 8
	1689	#define STORED_SEQS_MASK (STORED_SEQS-1)
	1690	#define ADVANCED_SEQS STORED_SEQS
	1691	seq_t sequences[STORED_SEQS];
	1692	int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS);
	1693	seqState_t seqState;
	1694	int seqNb;
	1695	size_t prefetchPos = (size_t)(op-prefixStart); /* track position relative to prefixStart */
	1696
	1697	dctx->fseEntropy = 1;
	1698	{ int i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
	1699	assert(dst != NULL);
	1700	assert(iend >= ip);
	1701	RETURN_ERROR_IF(
	1702	ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)),
	1703	corruption_detected, "");
	1704	ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
	1705	ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
	1706	ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
	1707
	1708	/* prepare in advance */
	1709	for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && (seqNb<seqAdvance); seqNb++) {
	1710	seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
	1711	prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd);
	1712	sequences[seqNb] = sequence;
	1713	}
	1714	RETURN_ERROR_IF(seqNb<seqAdvance, corruption_detected, "");
	1715
	1716	/* decompress without stomping litBuffer */
	1717	for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && (seqNb < nbSeq); seqNb++) {
	1718	seq_t sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
	1719	size_t oneSeqSize;
	1720
	1721	if (dctx->litBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength > dctx->litBufferEnd)
	1722	{
	1723	/* lit buffer is reaching split point, empty out the first buffer and transition to litExtraBuffer */
	1724	const size_t leftoverLit = dctx->litBufferEnd - litPtr;
	1725	if (leftoverLit)
	1726	{
	1727	RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
	1728	ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
	1729	sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit;
	1730	op += leftoverLit;
	1731	}
	1732	litPtr = dctx->litExtraBuffer;
	1733	litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
	1734	dctx->litBufferLocation = ZSTD_not_in_dst;
	1735	oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
	1736	#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
	1737	assert(!ZSTD_isError(oneSeqSize));
	1738	if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart);
	1739	#endif
	1740	if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
	1741
	1742	prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd);
	1743	sequences[seqNb & STORED_SEQS_MASK] = sequence;
	1744	op += oneSeqSize;
	1745	}
	1746	else
	1747	{
	1748	/* lit buffer is either wholly contained in first or second split, or not split at all*/
	1749	oneSeqSize = dctx->litBufferLocation == ZSTD_split ?
	1750	ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENGTH, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) :
	1751	ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
	1752	#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
	1753	assert(!ZSTD_isError(oneSeqSize));
	1754	if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart);
	1755	#endif
	1756	if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
	1757
	1758	prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd);
	1759	sequences[seqNb & STORED_SEQS_MASK] = sequence;
	1760	op += oneSeqSize;
	1761	}
	1762	}
	1763	RETURN_ERROR_IF(seqNb<nbSeq, corruption_detected, "");
	1764
	1765	/* finish queue */
	1766	seqNb -= seqAdvance;
	1767	for ( ; seqNb<nbSeq ; seqNb++) {
	1768	seq_t *sequence = &(sequences[seqNb&STORED_SEQS_MASK]);
	1769	if (dctx->litBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd)
	1770	{
	1771	const size_t leftoverLit = dctx->litBufferEnd - litPtr;
	1772	if (leftoverLit)
	1773	{
	1774	RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
	1775	ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
	1776	sequence->litLength -= leftoverLit;
	1777	op += leftoverLit;
	1778	}
	1779	litPtr = dctx->litExtraBuffer;
	1780	litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
	1781	dctx->litBufferLocation = ZSTD_not_in_dst;
	1782	{
	1783	size_t const oneSeqSize = ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
	1784	#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
	1785	assert(!ZSTD_isError(oneSeqSize));
	1786	if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart);
	1787	#endif
	1788	if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
	1789	op += oneSeqSize;
	1790	}
	1791	}
	1792	else
	1793	{
	1794	size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ?
	1795	ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) :
	1796	ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
	1797	#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
	1798	assert(!ZSTD_isError(oneSeqSize));
	1799	if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart);
	1800	#endif
	1801	if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
	1802	op += oneSeqSize;
	1803	}
	1804	}
	1805
	1806	/* save reps for next block */
	1807	{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
	1808	}
	1809
	1810	/* last literal segment */
	1811	if (dctx->litBufferLocation == ZSTD_split) /* first deplete literal buffer in dst, then copy litExtraBuffer */
	1812	{
	1813	size_t const lastLLSize = litBufferEnd - litPtr;
	1814	RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, "");
	1815	if (op != NULL) {
	1816	ZSTD_memmove(op, litPtr, lastLLSize);
	1817	op += lastLLSize;
	1818	}
	1819	litPtr = dctx->litExtraBuffer;
	1820	litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
	1821	}
	1822	{ size_t const lastLLSize = litBufferEnd - litPtr;
	1823	RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
	1824	if (op != NULL) {
	1825	ZSTD_memmove(op, litPtr, lastLLSize);
	1826	op += lastLLSize;
	1827	}
	1828	}
	1829
	1830	return op-ostart;
	1831	}
	1832
	1833	static size_t
	1834	ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx,
	1835	void* dst, size_t maxDstSize,
	1836	const void* seqStart, size_t seqSize, int nbSeq,
	1837	const ZSTD_longOffset_e isLongOffset,
	1838	const int frame)
	1839	{
	1840	return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
	1841	}
	1842	#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
	1843
	1844
	1845
	1846	#if DYNAMIC_BMI2
	1847
	1848	#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
	1849	static BMI2_TARGET_ATTRIBUTE size_t
	1850	DONT_VECTORIZE
	1851	ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx,
	1852	void* dst, size_t maxDstSize,
	1853	const void* seqStart, size_t seqSize, int nbSeq,
	1854	const ZSTD_longOffset_e isLongOffset,
	1855	const int frame)
	1856	{
	1857	return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
	1858	}
	1859	static BMI2_TARGET_ATTRIBUTE size_t
	1860	DONT_VECTORIZE
	1861	ZSTD_decompressSequencesSplitLitBuffer_bmi2(ZSTD_DCtx* dctx,
	1862	void* dst, size_t maxDstSize,
	1863	const void* seqStart, size_t seqSize, int nbSeq,
	1864	const ZSTD_longOffset_e isLongOffset,
	1865	const int frame)
	1866	{
	1867	return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
	1868	}
	1869	#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
	1870
	1871	#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
	1872	static BMI2_TARGET_ATTRIBUTE size_t
	1873	ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx,
	1874	void* dst, size_t maxDstSize,
	1875	const void* seqStart, size_t seqSize, int nbSeq,
	1876	const ZSTD_longOffset_e isLongOffset,
	1877	const int frame)
	1878	{
	1879	return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
	1880	}
	1881	#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
	1882
	1883	#endif /* DYNAMIC_BMI2 */
	1884
	1885	typedef size_t (*ZSTD_decompressSequences_t)(
	1886	ZSTD_DCtx* dctx,
	1887	void* dst, size_t maxDstSize,
	1888	const void* seqStart, size_t seqSize, int nbSeq,
	1889	const ZSTD_longOffset_e isLongOffset,
	1890	const int frame);
	1891
	1892	#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
	1893	static size_t
	1894	ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize,
	1895	const void* seqStart, size_t seqSize, int nbSeq,
	1896	const ZSTD_longOffset_e isLongOffset,
	1897	const int frame)
	1898	{
	1899	DEBUGLOG(5, "ZSTD_decompressSequences");
	1900	#if DYNAMIC_BMI2
	1901	if (ZSTD_DCtx_get_bmi2(dctx)) {
	1902	return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
	1903	}
	1904	#endif
	1905	return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
	1906	}
	1907	static size_t
	1908	ZSTD_decompressSequencesSplitLitBuffer(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize,
	1909	const void* seqStart, size_t seqSize, int nbSeq,
	1910	const ZSTD_longOffset_e isLongOffset,
	1911	const int frame)
	1912	{
	1913	DEBUGLOG(5, "ZSTD_decompressSequencesSplitLitBuffer");
	1914	#if DYNAMIC_BMI2
	1915	if (ZSTD_DCtx_get_bmi2(dctx)) {
	1916	return ZSTD_decompressSequencesSplitLitBuffer_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
	1917	}
	1918	#endif
	1919	return ZSTD_decompressSequencesSplitLitBuffer_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
	1920	}
	1921	#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
	1922
	1923
	1924	#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
	1925	/* ZSTD_decompressSequencesLong() :
	1926	* decompression function triggered when a minimum share of offsets is considered "long",
	1927	* aka out of cache.
	1928	* note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes meaning "farther than memory cache distance".
	1929	* This function will try to mitigate main memory latency through the use of prefetching */
	1930	static size_t
	1931	ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx,
	1932	void* dst, size_t maxDstSize,
	1933	const void* seqStart, size_t seqSize, int nbSeq,
	1934	const ZSTD_longOffset_e isLongOffset,
	1935	const int frame)
	1936	{
	1937	DEBUGLOG(5, "ZSTD_decompressSequencesLong");
	1938	#if DYNAMIC_BMI2
	1939	if (ZSTD_DCtx_get_bmi2(dctx)) {
	1940	return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
	1941	}
	1942	#endif
	1943	return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
	1944	}
	1945	#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
	1946
	1947
	1948
	1949	#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
	1950	!defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
	1951	/* ZSTD_getLongOffsetsShare() :
	1952	* condition : offTable must be valid
	1953	* @return : "share" of long offsets (arbitrarily defined as > (1<<23))
	1954	* compared to maximum possible of (1<<OffFSELog) */
	1955	static unsigned
	1956	ZSTD_getLongOffsetsShare(const ZSTD_seqSymbol* offTable)
	1957	{
	1958	const void* ptr = offTable;
	1959	U32 const tableLog = ((const ZSTD_seqSymbol_header*)ptr)[0].tableLog;
	1960	const ZSTD_seqSymbol* table = offTable + 1;
	1961	U32 const max = 1 << tableLog;
	1962	U32 u, total = 0;
	1963	DEBUGLOG(5, "ZSTD_getLongOffsetsShare: (tableLog=%u)", tableLog);
	1964
	1965	assert(max <= (1 << OffFSELog)); /* max not too large */
	1966	for (u=0; u<max; u++) {
	1967	if (table[u].nbAdditionalBits > 22) total += 1;
	1968	}
	1969
	1970	assert(tableLog <= OffFSELog);
	1971	total <<= (OffFSELog - tableLog); /* scale to OffFSELog */
	1972
	1973	return total;
	1974	}
	1975	#endif
	1976
	1977	size_t
	1978	ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
	1979	void* dst, size_t dstCapacity,
	1980	const void* src, size_t srcSize, const int frame, const streaming_operation streaming)
	1981	{ /* blockType == blockCompressed */
	1982	const BYTE* ip = (const BYTE*)src;
	1983	/* isLongOffset must be true if there are long offsets.
	1984	* Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN.
	1985	* We don't expect that to be the case in 64-bit mode.
	1986	* In block mode, window size is not known, so we have to be conservative.
	1987	* (note: but it could be evaluated from current-lowLimit)
	1988	*/
	1989	ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame \|\| (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN))));
	1990	DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize);
	1991
	1992	RETURN_ERROR_IF(srcSize >= ZSTD_BLOCKSIZE_MAX, srcSize_wrong, "");
	1993
	1994	/* Decode literals section */
	1995	{ size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming);
	1996	DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize);
	1997	if (ZSTD_isError(litCSize)) return litCSize;
	1998	ip += litCSize;
	1999	srcSize -= litCSize;
	2000	}
	2001
	2002	/* Build Decoding Tables */
	2003	{
	2004	/* These macros control at build-time which decompressor implementation
	2005	* we use. If neither is defined, we do some inspection and dispatch at
	2006	* runtime.
	2007	*/
	2008	#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
	2009	!defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
	2010	int usePrefetchDecoder = dctx->ddictIsCold;
	2011	#endif
	2012	int nbSeq;
	2013	size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize);
	2014	if (ZSTD_isError(seqHSize)) return seqHSize;
	2015	ip += seqHSize;
	2016	srcSize -= seqHSize;
	2017
	2018	RETURN_ERROR_IF(dst == NULL && nbSeq > 0, dstSize_tooSmall, "NULL not handled");
	2019
	2020	#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
	2021	!defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
	2022	if ( !usePrefetchDecoder
	2023	&& (!frame \|\| (dctx->fParams.windowSize > (1<<24)))
	2024	&& (nbSeq>ADVANCED_SEQS) ) { /* could probably use a larger nbSeq limit */
	2025	U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr);
	2026	U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */
	2027	usePrefetchDecoder = (shareLongOffsets >= minShare);
	2028	}
	2029	#endif
	2030
	2031	dctx->ddictIsCold = 0;
	2032
	2033	#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
	2034	!defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
	2035	if (usePrefetchDecoder)
	2036	#endif
	2037	#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
	2038	return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame);
	2039	#endif
	2040
	2041	#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
	2042	/* else */
	2043	if (dctx->litBufferLocation == ZSTD_split)
	2044	return ZSTD_decompressSequencesSplitLitBuffer(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame);
	2045	else
	2046	return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame);
	2047	#endif
	2048	}
	2049	}
	2050
	2051
	2052	void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize)
	2053	{
	2054	if (dst != dctx->previousDstEnd && dstSize > 0) { /* not contiguous */
	2055	dctx->dictEnd = dctx->previousDstEnd;
	2056	dctx->virtualStart = (const char)dst - ((const char)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart));
	2057	dctx->prefixStart = dst;
	2058	dctx->previousDstEnd = dst;
	2059	}
	2060	}
	2061
	2062
	2063	size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
	2064	void* dst, size_t dstCapacity,
	2065	const void* src, size_t srcSize)
	2066	{
	2067	size_t dSize;
	2068	ZSTD_checkContinuity(dctx, dst, dstCapacity);
	2069	dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0, not_streaming);
	2070	dctx->previousDstEnd = (char*)dst + dSize;
	2071	return dSize;
	2072	}

stage1/zstd/lib/decompress/zstd_decompress_block.h created+68

...	...	@@ -0,0 +1,68 @@
	1	/*
	2	* Copyright (c) Yann Collet, Facebook, Inc.
	3	* All rights reserved.
	4	*
	5	* This source code is licensed under both the BSD-style license (found in the
	6	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	7	* in the COPYING file in the root directory of this source tree).
	8	* You may select, at your option, one of the above-listed licenses.
	9	*/
	10
	11
	12	#ifndef ZSTD_DEC_BLOCK_H
	13	#define ZSTD_DEC_BLOCK_H
	14
	15	/-******************************************************
	16	* Dependencies
	17	*********************************************************/
	18	#include "../common/zstd_deps.h" /* size_t */
	19	#include "../zstd.h" /* DCtx, and some public functions */
	20	#include "../common/zstd_internal.h" /* blockProperties_t, and some public functions */
	21	#include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */
	22
	23
	24	/* === Prototypes === */
	25
	26	/* note: prototypes already published within `zstd.h` :
	27	* ZSTD_decompressBlock()
	28	*/
	29
	30	/* note: prototypes already published within `zstd_internal.h` :
	31	* ZSTD_getcBlockSize()
	32	* ZSTD_decodeSeqHeaders()
	33	*/
	34
	35
	36	/* Streaming state is used to inform allocation of the literal buffer */
	37	typedef enum {
	38	not_streaming = 0,
	39	is_streaming = 1
	40	} streaming_operation;
	41
	42	/* ZSTD_decompressBlock_internal() :
	43	* decompress block, starting at `src`,
	44	* into destination buffer `dst`.
	45	* @return : decompressed block size,
	46	* or an error code (which can be tested using ZSTD_isError())
	47	*/
	48	size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
	49	void* dst, size_t dstCapacity,
	50	const void* src, size_t srcSize, const int frame, const streaming_operation streaming);
	51
	52	/* ZSTD_buildFSETable() :
	53	* generate FSE decoding table for one symbol (ll, ml or off)
	54	* this function must be called with valid parameters only
	55	* (dt is large enough, normalizedCounter distribution total is a power of 2, max is within range, etc.)
	56	* in which case it cannot fail.
	57	* The workspace must be 4-byte aligned and at least ZSTD_BUILD_FSE_TABLE_WKSP_SIZE bytes, which is
	58	* defined in zstd_decompress_internal.h.
	59	* Internal use only.
	60	*/
	61	void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
	62	const short* normalizedCounter, unsigned maxSymbolValue,
	63	const U32* baseValue, const U8* nbAdditionalBits,
	64	unsigned tableLog, void* wksp, size_t wkspSize,
	65	int bmi2);
	66
	67
	68	#endif /* ZSTD_DEC_BLOCK_H */

stage1/zstd/lib/decompress/zstd_decompress_internal.h created+236

...	...	@@ -0,0 +1,236 @@
	1	/*
	2	* Copyright (c) Yann Collet, Facebook, Inc.
	3	* All rights reserved.
	4	*
	5	* This source code is licensed under both the BSD-style license (found in the
	6	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	7	* in the COPYING file in the root directory of this source tree).
	8	* You may select, at your option, one of the above-listed licenses.
	9	*/
	10
	11
	12	/* zstd_decompress_internal:
	13	* objects and definitions shared within lib/decompress modules */
	14
	15	#ifndef ZSTD_DECOMPRESS_INTERNAL_H
	16	#define ZSTD_DECOMPRESS_INTERNAL_H
	17
	18
	19	/-******************************************************
	20	* Dependencies
	21	*********************************************************/
	22	#include "../common/mem.h" /* BYTE, U16, U32 */
	23	#include "../common/zstd_internal.h" /* constants : MaxLL, MaxML, MaxOff, LLFSELog, etc. */
	24
	25
	26
	27	/-******************************************************
	28	* Constants
	29	*********************************************************/
	30	static UNUSED_ATTR const U32 LL_base[MaxLL+1] = {
	31	0, 1, 2, 3, 4, 5, 6, 7,
	32	8, 9, 10, 11, 12, 13, 14, 15,
	33	16, 18, 20, 22, 24, 28, 32, 40,
	34	48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000,
	35	0x2000, 0x4000, 0x8000, 0x10000 };
	36
	37	static UNUSED_ATTR const U32 OF_base[MaxOff+1] = {
	38	0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D,
	39	0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD,
	40	0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
	41	0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD };
	42
	43	static UNUSED_ATTR const U8 OF_bits[MaxOff+1] = {
	44	0, 1, 2, 3, 4, 5, 6, 7,
	45	8, 9, 10, 11, 12, 13, 14, 15,
	46	16, 17, 18, 19, 20, 21, 22, 23,
	47	24, 25, 26, 27, 28, 29, 30, 31 };
	48
	49	static UNUSED_ATTR const U32 ML_base[MaxML+1] = {
	50	3, 4, 5, 6, 7, 8, 9, 10,
	51	11, 12, 13, 14, 15, 16, 17, 18,
	52	19, 20, 21, 22, 23, 24, 25, 26,
	53	27, 28, 29, 30, 31, 32, 33, 34,
	54	35, 37, 39, 41, 43, 47, 51, 59,
	55	67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803,
	56	0x1003, 0x2003, 0x4003, 0x8003, 0x10003 };
	57
	58
	59	/-******************************************************
	60	* Decompression types
	61	*********************************************************/
	62	typedef struct {
	63	U32 fastMode;
	64	U32 tableLog;
	65	} ZSTD_seqSymbol_header;
	66
	67	typedef struct {
	68	U16 nextState;
	69	BYTE nbAdditionalBits;
	70	BYTE nbBits;
	71	U32 baseValue;
	72	} ZSTD_seqSymbol;
	73
	74	#define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log)))
	75
	76	#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64))
	77	#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32))
	78
	79	typedef struct {
	80	ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */
	81	ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */
	82	ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */
	83	HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
	84	U32 rep[ZSTD_REP_NUM];
	85	U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32];
	86	} ZSTD_entropyDTables_t;
	87
	88	typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
	89	ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock,
	90	ZSTDds_decompressLastBlock, ZSTDds_checkChecksum,
	91	ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
	92
	93	typedef enum { zdss_init=0, zdss_loadHeader,
	94	zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
	95
	96	typedef enum {
	97	ZSTD_use_indefinitely = -1, /* Use the dictionary indefinitely */
	98	ZSTD_dont_use = 0, /* Do not use the dictionary (if one exists free it) */
	99	ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */
	100	} ZSTD_dictUses_e;
	101
	102	/* Hashset for storing references to multiple ZSTD_DDict within ZSTD_DCtx */
	103	typedef struct {
	104	const ZSTD_DDict** ddictPtrTable;
	105	size_t ddictPtrTableSize;
	106	size_t ddictPtrCount;
	107	} ZSTD_DDictHashSet;
	108
	109	#ifndef ZSTD_DECODER_INTERNAL_BUFFER
	110	# define ZSTD_DECODER_INTERNAL_BUFFER (1 << 16)
	111	#endif
	112
	113	#define ZSTD_LBMIN 64
	114	#define ZSTD_LBMAX (128 << 10)
	115
	116	/* extra buffer, compensates when dst is not large enough to store litBuffer */
	117	#define ZSTD_LITBUFFEREXTRASIZE BOUNDED(ZSTD_LBMIN, ZSTD_DECODER_INTERNAL_BUFFER, ZSTD_LBMAX)
	118
	119	typedef enum {
	120	ZSTD_not_in_dst = 0, /* Stored entirely within litExtraBuffer */
	121	ZSTD_in_dst = 1, /* Stored entirely within dst (in memory after current output write) */
	122	ZSTD_split = 2 /* Split between litExtraBuffer and dst */
	123	} ZSTD_litLocation_e;
	124
	125	struct ZSTD_DCtx_s
	126	{
	127	const ZSTD_seqSymbol* LLTptr;
	128	const ZSTD_seqSymbol* MLTptr;
	129	const ZSTD_seqSymbol* OFTptr;
	130	const HUF_DTable* HUFptr;
	131	ZSTD_entropyDTables_t entropy;
	132	U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */
	133	const void* previousDstEnd; /* detect continuity */
	134	const void* prefixStart; /* start of current segment */
	135	const void* virtualStart; /* virtual start of previous segment if it was just before current one */
	136	const void* dictEnd; /* end of previous segment */
	137	size_t expected;
	138	ZSTD_frameHeader fParams;
	139	U64 processedCSize;
	140	U64 decodedSize;
	141	blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */
	142	ZSTD_dStage stage;
	143	U32 litEntropy;
	144	U32 fseEntropy;
	145	XXH64_state_t xxhState;
	146	size_t headerSize;
	147	ZSTD_format_e format;
	148	ZSTD_forceIgnoreChecksum_e forceIgnoreChecksum; /* User specified: if == 1, will ignore checksums in compressed frame. Default == 0 */
	149	U32 validateChecksum; /* if == 1, will validate checksum. Is == 1 if (fParams.checksumFlag == 1) and (forceIgnoreChecksum == 0). */
	150	const BYTE* litPtr;
	151	ZSTD_customMem customMem;
	152	size_t litSize;
	153	size_t rleSize;
	154	size_t staticSize;
	155	#if DYNAMIC_BMI2 != 0
	156	int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
	157	#endif
	158
	159	/* dictionary */
	160	ZSTD_DDict* ddictLocal;
	161	const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */
	162	U32 dictID;
	163	int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */
	164	ZSTD_dictUses_e dictUses;
	165	ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */
	166	ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */
	167
	168	/* streaming */
	169	ZSTD_dStreamStage streamStage;
	170	char* inBuff;
	171	size_t inBuffSize;
	172	size_t inPos;
	173	size_t maxWindowSize;
	174	char* outBuff;
	175	size_t outBuffSize;
	176	size_t outStart;
	177	size_t outEnd;
	178	size_t lhSize;
	179	#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
	180	void* legacyContext;
	181	U32 previousLegacyVersion;
	182	U32 legacyVersion;
	183	#endif
	184	U32 hostageByte;
	185	int noForwardProgress;
	186	ZSTD_bufferMode_e outBufferMode;
	187	ZSTD_outBuffer expectedOutBuffer;
	188
	189	/* workspace */
	190	BYTE* litBuffer;
	191	const BYTE* litBufferEnd;
	192	ZSTD_litLocation_e litBufferLocation;
	193	BYTE litExtraBuffer[ZSTD_LITBUFFEREXTRASIZE + WILDCOPY_OVERLENGTH]; /* literal buffer can be split between storage within dst and within this scratch buffer */
	194	BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
	195
	196	size_t oversizedDuration;
	197
	198	#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
	199	void const* dictContentBeginForFuzzing;
	200	void const* dictContentEndForFuzzing;
	201	#endif
	202
	203	/* Tracing */
	204	#if ZSTD_TRACE
	205	ZSTD_TraceCtx traceCtx;
	206	#endif
	207	}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
	208
	209	MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) {
	210	#if DYNAMIC_BMI2 != 0
	211	return dctx->bmi2;
	212	#else
	213	(void)dctx;
	214	return 0;
	215	#endif
	216	}
	217
	218	/-******************************************************
	219	* Shared internal functions
	220	*********************************************************/
	221
	222	/*! ZSTD_loadDEntropy() :
	223	* dict : must point at beginning of a valid zstd dictionary.
	224	* @return : size of dictionary header (size of magic number + dict ID + entropy tables) */
	225	size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
	226	const void* const dict, size_t const dictSize);
	227
	228	/*! ZSTD_checkContinuity() :
	229	* check if next `dst` follows previous position, where decompression ended.
	230	* If yes, do nothing (continue on current segment).
	231	* If not, classify previous segment as "external dictionary", and start a new segment.
	232	* This function cannot fail. */
	233	void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize);
	234
	235
	236	#endif /* ZSTD_DECOMPRESS_INTERNAL_H */

stage1/zstd/lib/zstd.h created+2575

...	...	@@ -0,0 +1,2575 @@
	1	/*
	2	* Copyright (c) Yann Collet, Facebook, Inc.
	3	* All rights reserved.
	4	*
	5	* This source code is licensed under both the BSD-style license (found in the
	6	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	7	* in the COPYING file in the root directory of this source tree).
	8	* You may select, at your option, one of the above-listed licenses.
	9	*/
	10	#if defined (__cplusplus)
	11	extern "C" {
	12	#endif
	13
	14	#ifndef ZSTD_H_235446
	15	#define ZSTD_H_235446
	16
	17	/* ====== Dependency ======*/
	18	#include <limits.h> /* INT_MAX */
	19	#include <stddef.h> /* size_t */
	20
	21
	22	/* ===== ZSTDLIB_API : control library symbols visibility ===== */
	23	#ifndef ZSTDLIB_VISIBLE
	24	# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
	25	# define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default")))
	26	# define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden")))
	27	# else
	28	# define ZSTDLIB_VISIBLE
	29	# define ZSTDLIB_HIDDEN
	30	# endif
	31	#endif
	32	#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
	33	# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE
	34	#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
	35	# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
	36	#else
	37	# define ZSTDLIB_API ZSTDLIB_VISIBLE
	38	#endif
	39
	40
	41	/*******************************************************************************
	42	Introduction
	43
	44	zstd, short for Zstandard, is a fast lossless compression algorithm, targeting
	45	real-time compression scenarios at zlib-level and better compression ratios.
	46	The zstd compression library provides in-memory compression and decompression
	47	functions.
	48
	49	The library supports regular compression levels from 1 up to ZSTD_maxCLevel(),
	50	which is currently 22. Levels >= 20, labeled `--ultra`, should be used with
	51	caution, as they require more memory. The library also offers negative
	52	compression levels, which extend the range of speed vs. ratio preferences.
	53	The lower the level, the faster the speed (at the cost of compression).
	54
	55	Compression can be done in:
	56	- a single step (described as Simple API)
	57	- a single step, reusing a context (described as Explicit context)
	58	- unbounded multiple steps (described as Streaming compression)
	59
	60	The compression ratio achievable on small data can be highly improved using
	61	a dictionary. Dictionary compression can be performed in:
	62	- a single step (described as Simple dictionary API)
	63	- a single step, reusing a dictionary (described as Bulk-processing
	64	dictionary API)
	65
	66	Advanced experimental functions can be accessed using
	67	`#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h.
	68
	69	Advanced experimental APIs should never be used with a dynamically-linked
	70	library. They are not "stable"; their definitions or signatures may change in
	71	the future. Only static linking is allowed.
	72	*******************************************************************************/
	73
	74	/------ Version ------/
	75	#define ZSTD_VERSION_MAJOR 1
	76	#define ZSTD_VERSION_MINOR 5
	77	#define ZSTD_VERSION_RELEASE 2
	78	#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR 100100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
	79
	80	/*! ZSTD_versionNumber() :
	81	* Return runtime library version, the value is (MAJOR100100 + MINOR100 + RELEASE). /
	82	ZSTDLIB_API unsigned ZSTD_versionNumber(void);
	83
	84	#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
	85	#define ZSTD_QUOTE(str) #str
	86	#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str)
	87	#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
	88
	89	/*! ZSTD_versionString() :
	90	* Return runtime library version, like "1.4.5". Requires v1.3.0+. */
	91	ZSTDLIB_API const char* ZSTD_versionString(void);
	92
	93	/* *************************************
	94	* Default constant
	95	***************************************/
	96	#ifndef ZSTD_CLEVEL_DEFAULT
	97	# define ZSTD_CLEVEL_DEFAULT 3
	98	#endif
	99
	100	/* *************************************
	101	* Constants
	102	***************************************/
	103
	104	/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */
	105	#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */
	106	#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */
	107	#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */
	108	#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0
	109
	110	#define ZSTD_BLOCKSIZELOG_MAX 17
	111	#define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX)
	112
	113
	114	/***************************************
	115	* Simple API
	116	***************************************/
	117	/*! ZSTD_compress() :
	118	* Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
	119	* Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
	120	* @return : compressed size written into `dst` (<= `dstCapacity),
	121	* or an error code if it fails (which can be tested using ZSTD_isError()). */
	122	ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
	123	const void* src, size_t srcSize,
	124	int compressionLevel);
	125
	126	/*! ZSTD_decompress() :
	127	* `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
	128	* `dstCapacity` is an upper bound of originalSize to regenerate.
	129	* If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
	130	* @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
	131	* or an errorCode if it fails (which can be tested using ZSTD_isError()). */
	132	ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
	133	const void* src, size_t compressedSize);
	134
	135	/*! ZSTD_getFrameContentSize() : requires v1.3.0+
	136	* `src` should point to the start of a ZSTD encoded frame.
	137	* `srcSize` must be at least as large as the frame header.
	138	* hint : any size >= `ZSTD_frameHeaderSize_max` is large enough.
	139	* @return : - decompressed size of `src` frame content, if known
	140	* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
	141	* - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small)
	142	* note 1 : a 0 return value means the frame is valid but "empty".
	143	* note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode.
	144	* When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
	145	* In which case, it's necessary to use streaming mode to decompress data.
	146	* Optionally, application can rely on some implicit limit,
	147	* as ZSTD_decompress() only needs an upper bound of decompressed size.
	148	* (For example, data could be necessarily cut into blocks <= 16 KB).
	149	* note 3 : decompressed size is always present when compression is completed using single-pass functions,
	150	* such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict().
	151	* note 4 : decompressed size can be very large (64-bits value),
	152	* potentially larger than what local system can handle as a single memory segment.
	153	* In which case, it's necessary to use streaming mode to decompress data.
	154	* note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified.
	155	* Always ensure return value fits within application's authorized limits.
	156	* Each application can set its own limits.
	157	* note 6 : This function replaces ZSTD_getDecompressedSize() */
	158	#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
	159	#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
	160	ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
	161
	162	/*! ZSTD_getDecompressedSize() :
	163	* NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize().
	164	* Both functions work the same way, but ZSTD_getDecompressedSize() blends
	165	* "empty", "unknown" and "error" results to the same return value (0),
	166	* while ZSTD_getFrameContentSize() gives them separate return values.
	167	* @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */
	168	ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
	169
	170	/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+
	171	* `src` should point to the start of a ZSTD frame or skippable frame.
	172	* `srcSize` must be >= first frame size
	173	* @return : the compressed size of the first frame starting at `src`,
	174	* suitable to pass as `srcSize` to `ZSTD_decompress` or similar,
	175	* or an error code if input is invalid */
	176	ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
	177
	178
	179	/====== Helper functions ======/
	180	#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 / : 0)) / this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
	181	ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /!< maximum compressed size in worst case single-pass scenario /
	182	ZSTDLIB_API unsigned ZSTD_isError(size_t code); /!< tells if a `size_t` function result is an error code /
	183	ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /!< provides readable string from an error code /
	184	ZSTDLIB_API int ZSTD_minCLevel(void); /!< minimum negative compression level allowed, requires v1.4.0+ /
	185	ZSTDLIB_API int ZSTD_maxCLevel(void); /!< maximum compression level available /
	186	ZSTDLIB_API int ZSTD_defaultCLevel(void); /!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ /
	187
	188
	189	/***************************************
	190	* Explicit context
	191	***************************************/
	192	/*= Compression context
	193	* When compressing many times,
	194	* it is recommended to allocate a context just once,
	195	* and re-use it for each successive compression operation.
	196	* This will make workload friendlier for system's memory.
	197	* Note : re-using context is just a speed / resource optimization.
	198	* It doesn't change the compression ratio, which remains identical.
	199	* Note 2 : In multi-threaded environments,
	200	* use one different context per thread for parallel execution.
	201	*/
	202	typedef struct ZSTD_CCtx_s ZSTD_CCtx;
	203	ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
	204	ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* accept NULL pointer */
	205
	206	/*! ZSTD_compressCCtx() :
	207	* Same as ZSTD_compress(), using an explicit ZSTD_CCtx.
	208	* Important : in order to behave similarly to `ZSTD_compress()`,
	209	* this function compresses at requested compression level,
	210	* __ignoring any other parameter__ .
	211	* If any advanced parameter was set using the advanced API,
	212	* they will all be reset. Only `compressionLevel` remains.
	213	*/
	214	ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx,
	215	void* dst, size_t dstCapacity,
	216	const void* src, size_t srcSize,
	217	int compressionLevel);
	218
	219	/*= Decompression context
	220	* When decompressing many times,
	221	* it is recommended to allocate a context only once,
	222	* and re-use it for each successive compression operation.
	223	* This will make workload friendlier for system's memory.
	224	* Use one context per thread for parallel execution. */
	225	typedef struct ZSTD_DCtx_s ZSTD_DCtx;
	226	ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
	227	ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /* accept NULL pointer */
	228
	229	/*! ZSTD_decompressDCtx() :
	230	* Same as ZSTD_decompress(),
	231	* requires an allocated ZSTD_DCtx.
	232	* Compatible with sticky parameters.
	233	*/
	234	ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
	235	void* dst, size_t dstCapacity,
	236	const void* src, size_t srcSize);
	237
	238
	239	/*********************************************
	240	* Advanced compression API (Requires v1.4.0+)
	241	**********************************************/
	242
	243	/* API design :
	244	* Parameters are pushed one by one into an existing context,
	245	* using ZSTD_CCtx_set*() functions.
	246	* Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame.
	247	* "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` !
	248	* __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ .
	249	*
	250	* It's possible to reset all parameters to "default" using ZSTD_CCtx_reset().
	251	*
	252	* This API supersedes all other "advanced" API entry points in the experimental section.
	253	* In the future, we expect to remove from experimental API entry points which are redundant with this API.
	254	*/
	255
	256
	257	/* Compression strategies, listed from fastest to strongest */
	258	typedef enum { ZSTD_fast=1,
	259	ZSTD_dfast=2,
	260	ZSTD_greedy=3,
	261	ZSTD_lazy=4,
	262	ZSTD_lazy2=5,
	263	ZSTD_btlazy2=6,
	264	ZSTD_btopt=7,
	265	ZSTD_btultra=8,
	266	ZSTD_btultra2=9
	267	/* note : new strategies _might_ be added in the future.
	268	Only the order (from fast to strong) is guaranteed */
	269	} ZSTD_strategy;
	270
	271	typedef enum {
	272
	273	/* compression parameters
	274	* Note: When compressing with a ZSTD_CDict these parameters are superseded
	275	* by the parameters used to construct the ZSTD_CDict.
	276	* See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */
	277	ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table.
	278	* Note that exact compression parameters are dynamically determined,
	279	* depending on both compression level and srcSize (when known).
	280	* Default level is ZSTD_CLEVEL_DEFAULT==3.
	281	* Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
	282	* Note 1 : it's possible to pass a negative compression level.
	283	* Note 2 : setting a level does not automatically set all other compression parameters
	284	* to default. Setting this will however eventually dynamically impact the compression
	285	* parameters which have not been manually set. The manually set
	286	* ones will 'stick'. */
	287	/* Advanced compression parameters :
	288	* It's possible to pin down compression parameters to some specific values.
	289	* In which case, these values are no longer dynamically selected by the compressor */
	290	ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2.
	291	* This will set a memory budget for streaming decompression,
	292	* with larger values requiring more memory
	293	* and typically compressing more.
	294	* Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
	295	* Special: value 0 means "use default windowLog".
	296	* Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
	297	* requires explicitly allowing such size at streaming decompression stage. */
	298	ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2.
	299	* Resulting memory usage is (1 << (hashLog+2)).
	300	* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
	301	* Larger tables improve compression ratio of strategies <= dFast,
	302	* and improve speed of strategies > dFast.
	303	* Special: value 0 means "use default hashLog". */
	304	ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2.
	305	* Resulting memory usage is (1 << (chainLog+2)).
	306	* Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
	307	* Larger tables result in better and slower compression.
	308	* This parameter is useless for "fast" strategy.
	309	* It's still useful when using "dfast" strategy,
	310	* in which case it defines a secondary probe table.
	311	* Special: value 0 means "use default chainLog". */
	312	ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2.
	313	* More attempts result in better and slower compression.
	314	* This parameter is useless for "fast" and "dFast" strategies.
	315	* Special: value 0 means "use default searchLog". */
	316	ZSTD_c_minMatch=105, /* Minimum size of searched matches.
	317	* Note that Zstandard can still find matches of smaller size,
	318	* it just tweaks its search algorithm to look for this size and larger.
	319	* Larger values increase compression and decompression speed, but decrease ratio.
	320	* Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX.
	321	* Note that currently, for all strategies < btopt, effective minimum is 4.
	322	* , for all strategies > fast, effective maximum is 6.
	323	* Special: value 0 means "use default minMatchLength". */
	324	ZSTD_c_targetLength=106, /* Impact of this field depends on strategy.
	325	* For strategies btopt, btultra & btultra2:
	326	* Length of Match considered "good enough" to stop search.
	327	* Larger values make compression stronger, and slower.
	328	* For strategy fast:
	329	* Distance between match sampling.
	330	* Larger values make compression faster, and weaker.
	331	* Special: value 0 means "use default targetLength". */
	332	ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition.
	333	* The higher the value of selected strategy, the more complex it is,
	334	* resulting in stronger and slower compression.
	335	* Special: value 0 means "use default strategy". */
	336	/* LDM mode parameters */
	337	ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching.
	338	* This parameter is designed to improve compression ratio
	339	* for large inputs, by finding large matches at long distance.
	340	* It increases memory usage and window size.
	341	* Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB
	342	* except when expressly set to a different value.
	343	* Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and
	344	* compression strategy >= ZSTD_btopt (== compression level 16+) */
	345	ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2.
	346	* Larger values increase memory usage and compression ratio,
	347	* but decrease compression speed.
	348	* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
	349	* default: windowlog - 7.
	350	* Special: value 0 means "automatically determine hashlog". */
	351	ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher.
	352	* Larger/too small values usually decrease compression ratio.
	353	* Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX.
	354	* Special: value 0 means "use default value" (default: 64). */
	355	ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution.
	356	* Larger values improve collision resolution but decrease compression speed.
	357	* The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX.
	358	* Special: value 0 means "use default value" (default: 3). */
	359	ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table.
	360	* Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN).
	361	* Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage.
	362	* Larger values improve compression speed.
	363	* Deviating far from default value will likely result in a compression ratio decrease.
	364	* Special: value 0 means "automatically determine hashRateLog". */
	365
	366	/* frame parameters */
	367	ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1)
	368	* Content size must be known at the beginning of compression.
	369	* This is automatically the case when using ZSTD_compress2(),
	370	* For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */
	371	ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */
	372	ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */
	373
	374	/* multi-threading parameters */
	375	/* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
	376	* Otherwise, trying to set any other value than default (0) will be a no-op and return an error.
	377	* In a situation where it's unknown if the linked library supports multi-threading or not,
	378	* setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property.
	379	*/
	380	ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel.
	381	* When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() :
	382	* ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
	383	* while compression is performed in parallel, within worker thread(s).
	384	* (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
	385	* in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
	386	* More workers improve speed, but also increase memory usage.
	387	* Default value is `0`, aka "single-threaded mode" : no worker is spawned,
	388	* compression is performed inside Caller's thread, and all invocations are blocking */
	389	ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1.
	390	* Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
	391	* 0 means default, which is dynamically determined based on compression parameters.
	392	* Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest.
	393	* The minimum size is automatically and transparently enforced. */
	394	ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size.
	395	* The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
	396	* It helps preserve compression ratio, while each job is compressed in parallel.
	397	* This value is enforced only when nbWorkers >= 1.
	398	* Larger values increase compression ratio, but decrease speed.
	399	* Possible values range from 0 to 9 :
	400	* - 0 means "default" : value will be determined by the library, depending on strategy
	401	* - 1 means "no overlap"
	402	* - 9 means "full overlap", using a full window size.
	403	* Each intermediate rank increases/decreases load size by a factor 2 :
	404	* 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default
	405	* default value varies between 6 and 9, depending on strategy */
	406
	407	/* note : additional experimental parameters are also available
	408	* within the experimental section of the API.
	409	* At the time of this writing, they include :
	410	* ZSTD_c_rsyncable
	411	* ZSTD_c_format
	412	* ZSTD_c_forceMaxWindow
	413	* ZSTD_c_forceAttachDict
	414	* ZSTD_c_literalCompressionMode
	415	* ZSTD_c_targetCBlockSize
	416	* ZSTD_c_srcSizeHint
	417	* ZSTD_c_enableDedicatedDictSearch
	418	* ZSTD_c_stableInBuffer
	419	* ZSTD_c_stableOutBuffer
	420	* ZSTD_c_blockDelimiters
	421	* ZSTD_c_validateSequences
	422	* ZSTD_c_useBlockSplitter
	423	* ZSTD_c_useRowMatchFinder
	424	* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
	425	* note : never ever use experimentalParam? names directly;
	426	* also, the enums values themselves are unstable and can still change.
	427	*/
	428	ZSTD_c_experimentalParam1=500,
	429	ZSTD_c_experimentalParam2=10,
	430	ZSTD_c_experimentalParam3=1000,
	431	ZSTD_c_experimentalParam4=1001,
	432	ZSTD_c_experimentalParam5=1002,
	433	ZSTD_c_experimentalParam6=1003,
	434	ZSTD_c_experimentalParam7=1004,
	435	ZSTD_c_experimentalParam8=1005,
	436	ZSTD_c_experimentalParam9=1006,
	437	ZSTD_c_experimentalParam10=1007,
	438	ZSTD_c_experimentalParam11=1008,
	439	ZSTD_c_experimentalParam12=1009,
	440	ZSTD_c_experimentalParam13=1010,
	441	ZSTD_c_experimentalParam14=1011,
	442	ZSTD_c_experimentalParam15=1012
	443	} ZSTD_cParameter;
	444
	445	typedef struct {
	446	size_t error;
	447	int lowerBound;
	448	int upperBound;
	449	} ZSTD_bounds;
	450
	451	/*! ZSTD_cParam_getBounds() :
	452	* All parameters must belong to an interval with lower and upper bounds,
	453	* otherwise they will either trigger an error or be automatically clamped.
	454	* @return : a structure, ZSTD_bounds, which contains
	455	* - an error status field, which must be tested using ZSTD_isError()
	456	* - lower and upper bounds, both inclusive
	457	*/
	458	ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam);
	459
	460	/*! ZSTD_CCtx_setParameter() :
	461	* Set one compression parameter, selected by enum ZSTD_cParameter.
	462	* All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds().
	463	* Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
	464	* Setting a parameter is generally only possible during frame initialization (before starting compression).
	465	* Exception : when using multi-threading mode (nbWorkers >= 1),
	466	* the following parameters can be updated _during_ compression (within same frame):
	467	* => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy.
	468	* new parameters will be active for next job only (after a flush()).
	469	* @return : an error code (which can be tested using ZSTD_isError()).
	470	*/
	471	ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value);
	472
	473	/*! ZSTD_CCtx_setPledgedSrcSize() :
	474	* Total input data size to be compressed as a single frame.
	475	* Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag.
	476	* This value will also be controlled at end of frame, and trigger an error if not respected.
	477	* @result : 0, or an error code (which can be tested with ZSTD_isError()).
	478	* Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame.
	479	* In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN.
	480	* ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame.
	481	* Note 2 : pledgedSrcSize is only valid once, for the next frame.
	482	* It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN.
	483	* Note 3 : Whenever all input data is provided and consumed in a single round,
	484	* for example with ZSTD_compress2(),
	485	* or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end),
	486	* this value is automatically overridden by srcSize instead.
	487	*/
	488	ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize);
	489
	490	typedef enum {
	491	ZSTD_reset_session_only = 1,
	492	ZSTD_reset_parameters = 2,
	493	ZSTD_reset_session_and_parameters = 3
	494	} ZSTD_ResetDirective;
	495
	496	/*! ZSTD_CCtx_reset() :
	497	* There are 2 different things that can be reset, independently or jointly :
	498	* - The session : will stop compressing current frame, and make CCtx ready to start a new one.
	499	* Useful after an error, or to interrupt any ongoing compression.
	500	* Any internal data not yet flushed is cancelled.
	501	* Compression parameters and dictionary remain unchanged.
	502	* They will be used to compress next frame.
	503	* Resetting session never fails.
	504	* - The parameters : changes all parameters back to "default".
	505	* This removes any reference to any dictionary too.
	506	* Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
	507	* otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
	508	* - Both : similar to resetting the session, followed by resetting parameters.
	509	*/
	510	ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset);
	511
	512	/*! ZSTD_compress2() :
	513	* Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API.
	514	* ZSTD_compress2() always starts a new frame.
	515	* Should cctx hold data from a previously unfinished frame, everything about it is forgotten.
	516	* - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
	517	* - The function is always blocking, returns when compression is completed.
	518	* Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
	519	* @return : compressed size written into `dst` (<= `dstCapacity),
	520	* or an error code if it fails (which can be tested using ZSTD_isError()).
	521	*/
	522	ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx,
	523	void* dst, size_t dstCapacity,
	524	const void* src, size_t srcSize);
	525
	526
	527	/***********************************************
	528	* Advanced decompression API (Requires v1.4.0+)
	529	************************************************/
	530
	531	/* The advanced API pushes parameters one by one into an existing DCtx context.
	532	* Parameters are sticky, and remain valid for all following frames
	533	* using the same DCtx context.
	534	* It's possible to reset parameters to default values using ZSTD_DCtx_reset().
	535	* Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream().
	536	* Therefore, no new decompression function is necessary.
	537	*/
	538
	539	typedef enum {
	540
	541	ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which
	542	* the streaming API will refuse to allocate memory buffer
	543	* in order to protect the host from unreasonable memory requirements.
	544	* This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
	545	* By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT).
	546	* Special: value 0 means "use default maximum windowLog". */
	547
	548	/* note : additional experimental parameters are also available
	549	* within the experimental section of the API.
	550	* At the time of this writing, they include :
	551	* ZSTD_d_format
	552	* ZSTD_d_stableOutBuffer
	553	* ZSTD_d_forceIgnoreChecksum
	554	* ZSTD_d_refMultipleDDicts
	555	* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
	556	* note : never ever use experimentalParam? names directly
	557	*/
	558	ZSTD_d_experimentalParam1=1000,
	559	ZSTD_d_experimentalParam2=1001,
	560	ZSTD_d_experimentalParam3=1002,
	561	ZSTD_d_experimentalParam4=1003
	562
	563	} ZSTD_dParameter;
	564
	565	/*! ZSTD_dParam_getBounds() :
	566	* All parameters must belong to an interval with lower and upper bounds,
	567	* otherwise they will either trigger an error or be automatically clamped.
	568	* @return : a structure, ZSTD_bounds, which contains
	569	* - an error status field, which must be tested using ZSTD_isError()
	570	* - both lower and upper bounds, inclusive
	571	*/
	572	ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam);
	573
	574	/*! ZSTD_DCtx_setParameter() :
	575	* Set one compression parameter, selected by enum ZSTD_dParameter.
	576	* All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds().
	577	* Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter).
	578	* Setting a parameter is only possible during frame initialization (before starting decompression).
	579	* @return : 0, or an error code (which can be tested using ZSTD_isError()).
	580	*/
	581	ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value);
	582
	583	/*! ZSTD_DCtx_reset() :
	584	* Return a DCtx to clean state.
	585	* Session and parameters can be reset jointly or separately.
	586	* Parameters can only be reset when no active frame is being decompressed.
	587	* @return : 0, or an error code, which can be tested with ZSTD_isError()
	588	*/
	589	ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset);
	590
	591
	592	/****************************
	593	* Streaming
	594	****************************/
	595
	596	typedef struct ZSTD_inBuffer_s {
	597	const void* src; /*< start of input buffer /
	598	size_t size; /*< size of input buffer /
	599	size_t pos; /*< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size /
	600	} ZSTD_inBuffer;
	601
	602	typedef struct ZSTD_outBuffer_s {
	603	void* dst; /*< start of output buffer /
	604	size_t size; /*< size of output buffer /
	605	size_t pos; /*< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size /
	606	} ZSTD_outBuffer;
	607
	608
	609
	610	/-**********************************************************************
	611	* Streaming compression - HowTo
	612	*
	613	* A ZSTD_CStream object is required to track streaming operation.
	614	* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
	615	* ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
	616	* It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory.
	617	*
	618	* For parallel execution, use one separate ZSTD_CStream per thread.
	619	*
	620	* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing.
	621	*
	622	* Parameters are sticky : when starting a new compression on the same context,
	623	* it will re-use the same sticky parameters as previous compression session.
	624	* When in doubt, it's recommended to fully initialize the context before usage.
	625	* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(),
	626	* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to
	627	* set more specific parameters, the pledged source size, or load a dictionary.
	628	*
	629	* Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to
	630	* consume input stream. The function will automatically update both `pos`
	631	* fields within `input` and `output`.
	632	* Note that the function may not consume the entire input, for example, because
	633	* the output buffer is already full, in which case `input.pos < input.size`.
	634	* The caller must check if input has been entirely consumed.
	635	* If not, the caller must make some room to receive more compressed data,
	636	* and then present again remaining input data.
	637	* note: ZSTD_e_continue is guaranteed to make some forward progress when called,
	638	* but doesn't guarantee maximal forward progress. This is especially relevant
	639	* when compressing with multiple threads. The call won't block if it can
	640	* consume some input, but if it can't it will wait for some, but not all,
	641	* output to be flushed.
	642	* @return : provides a minimum amount of data remaining to be flushed from internal buffers
	643	* or an error code, which can be tested using ZSTD_isError().
	644	*
	645	* At any moment, it's possible to flush whatever data might remain stuck within internal buffer,
	646	* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated.
	647	* Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0).
	648	* In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush.
	649	* You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the
	650	* operation.
	651	* note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will
	652	* block until the flush is complete or the output buffer is full.
	653	* @return : 0 if internal buffers are entirely flushed,
	654	* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
	655	* or an error code, which can be tested using ZSTD_isError().
	656	*
	657	* Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame.
	658	* It will perform a flush and write frame epilogue.
	659	* The epilogue is required for decoders to consider a frame completed.
	660	* flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush.
	661	* You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to
	662	* start a new frame.
	663	* note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will
	664	* block until the flush is complete or the output buffer is full.
	665	* @return : 0 if frame fully completed and fully flushed,
	666	* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size),
	667	* or an error code, which can be tested using ZSTD_isError().
	668	*
	669	* *******************************************************************/
	670
	671	typedef ZSTD_CCtx ZSTD_CStream; /*< CCtx and CStream are now effectively same object (>= v1.3.0) /
	672	/* Continue to distinguish them for compatibility with older versions <= v1.2.0 */
	673	/===== ZSTD_CStream management functions =====/
	674	ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
	675	ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); /* accept NULL pointer */
	676
	677	/===== Streaming compression functions =====/
	678	typedef enum {
	679	ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
	680	ZSTD_e_flush=1, /* flush any data provided so far,
	681	* it creates (at least) one new block, that can be decoded immediately on reception;
	682	* frame will continue: any future data can still reference previously compressed data, improving compression.
	683	* note : multithreaded compression will block to flush as much output as possible. */
	684	ZSTD_e_end=2 /* flush any remaining data _and_ close current frame.
	685	* note that frame is only closed after compressed data is fully flushed (return value == 0).
	686	* After that point, any additional data starts a new frame.
	687	* note : each frame is independent (does not reference any content from previous frame).
	688	: note : multithreaded compression will block to flush as much output as possible. */
	689	} ZSTD_EndDirective;
	690
	691	/*! ZSTD_compressStream2() : Requires v1.4.0+
	692	* Behaves about the same as ZSTD_compressStream, with additional control on end directive.
	693	* - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
	694	* - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
	695	* - output->pos must be <= dstCapacity, input->pos must be <= srcSize
	696	* - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit.
	697	* - endOp must be a valid directive
	698	* - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller.
	699	* - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available,
	700	* and then immediately returns, just indicating that there is some data remaining to be flushed.
	701	* The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte.
	702	* - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking.
	703	* - @return provides a minimum amount of data remaining to be flushed from internal buffers
	704	* or an error code, which can be tested using ZSTD_isError().
	705	* if @return != 0, flush is not fully completed, there is still some data left within internal buffers.
	706	* This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers.
	707	* For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed.
	708	* - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0),
	709	* only ZSTD_e_end or ZSTD_e_flush operations are allowed.
	710	* Before starting a new compression job, or changing compression parameters,
	711	* it is required to fully flush internal buffers.
	712	*/
	713	ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
	714	ZSTD_outBuffer* output,
	715	ZSTD_inBuffer* input,
	716	ZSTD_EndDirective endOp);
	717
	718
	719	/* These buffer sizes are softly recommended.
	720	* They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output.
	721	* Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(),
	722	* reducing the amount of memory shuffling and buffering, resulting in minor performance savings.
	723	*
	724	* However, note that these recommendations are from the perspective of a C caller program.
	725	* If the streaming interface is invoked from some other language,
	726	* especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo,
	727	* a major performance rule is to reduce crossing such interface to an absolute minimum.
	728	* It's not rare that performance ends being spent more into the interface, rather than compression itself.
	729	* In which cases, prefer using large buffers, as large as practical,
	730	* for both input and output, to reduce the nb of roundtrips.
	731	*/
	732	ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /*< recommended size for input buffer /
	733	ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /*< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. /
	734
	735
	736	/* *****************************************************************************
	737	* This following is a legacy streaming API, available since v1.0+ .
	738	* It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2().
	739	* It is redundant, but remains fully supported.
	740	* Streaming in combination with advanced parameters and dictionary compression
	741	* can only be used through the new API.
	742	******************************************************************************/
	743
	744	/*!
	745	* Equivalent to:
	746	*
	747	* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
	748	* ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
	749	* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
	750	*/
	751	ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
	752	/*!
	753	* Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
	754	* NOTE: The return value is different. ZSTD_compressStream() returns a hint for
	755	* the next read size (if non-zero and not an error). ZSTD_compressStream2()
	756	* returns the minimum nb of bytes left to flush (if non-zero and not an error).
	757	*/
	758	ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
	759	/! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). /
	760	ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
	761	/! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). /
	762	ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
	763
	764
	765	/-**************************************************************************
	766	* Streaming decompression - HowTo
	767	*
	768	* A ZSTD_DStream object is required to track streaming operations.
	769	* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
	770	* ZSTD_DStream objects can be re-used multiple times.
	771	*
	772	* Use ZSTD_initDStream() to start a new decompression operation.
	773	* @return : recommended first input size
	774	* Alternatively, use advanced API to set specific properties.
	775	*
	776	* Use ZSTD_decompressStream() repetitively to consume your input.
	777	* The function will update both `pos` fields.
	778	* If `input.pos < input.size`, some input has not been consumed.
	779	* It's up to the caller to present again remaining data.
	780	* The function tries to flush all data decoded immediately, respecting output buffer size.
	781	* If `output.pos < output.size`, decoder has flushed everything it could.
	782	* But if `output.pos == output.size`, there might be some data left within internal buffers.,
	783	* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
	784	* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
	785	* @return : 0 when a frame is completely decoded and fully flushed,
	786	* or an error code, which can be tested using ZSTD_isError(),
	787	* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
	788	* the return value is a suggested next input size (just a hint for better latency)
	789	* that will never request more than the remaining frame size.
	790	* *******************************************************************************/
	791
	792	typedef ZSTD_DCtx ZSTD_DStream; /*< DCtx and DStream are now effectively same object (>= v1.3.0) /
	793	/* For compatibility with versions <= v1.2.0, prefer differentiating them. */
	794	/===== ZSTD_DStream management functions =====/
	795	ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
	796	ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer */
	797
	798	/===== Streaming decompression functions =====/
	799
	800	/* This function is redundant with the advanced API and equivalent to:
	801	*
	802	* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
	803	* ZSTD_DCtx_refDDict(zds, NULL);
	804	*/
	805	ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
	806
	807	ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
	808
	809	ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /!< recommended size for input buffer /
	810	ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. /
	811
	812
	813	/**************************
	814	* Simple dictionary API
	815	***************************/
	816	/*! ZSTD_compress_usingDict() :
	817	* Compression at an explicit compression level using a Dictionary.
	818	* A dictionary can be any arbitrary data segment (also called a prefix),
	819	* or a buffer with specified information (see zdict.h).
	820	* Note : This function loads the dictionary, resulting in significant startup delay.
	821	* It's intended for a dictionary used only once.
	822	* Note 2 : When `dict == NULL \|\| dictSize < 8` no dictionary is used. */
	823	ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
	824	void* dst, size_t dstCapacity,
	825	const void* src, size_t srcSize,
	826	const void* dict,size_t dictSize,
	827	int compressionLevel);
	828
	829	/*! ZSTD_decompress_usingDict() :
	830	* Decompression using a known Dictionary.
	831	* Dictionary must be identical to the one used during compression.
	832	* Note : This function loads the dictionary, resulting in significant startup delay.
	833	* It's intended for a dictionary used only once.
	834	* Note : When `dict == NULL \|\| dictSize < 8` no dictionary is used. */
	835	ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
	836	void* dst, size_t dstCapacity,
	837	const void* src, size_t srcSize,
	838	const void* dict,size_t dictSize);
	839
	840
	841	/***********************************
	842	* Bulk processing dictionary API
	843	**********************************/
	844	typedef struct ZSTD_CDict_s ZSTD_CDict;
	845
	846	/*! ZSTD_createCDict() :
	847	* When compressing multiple messages or blocks using the same dictionary,
	848	* it's recommended to digest the dictionary only once, since it's a costly operation.
	849	* ZSTD_createCDict() will create a state from digesting a dictionary.
	850	* The resulting state can be used for future compression operations with very limited startup cost.
	851	* ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
	852	* @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict.
	853	* Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content.
	854	* Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer,
	855	* in which case the only thing that it transports is the @compressionLevel.
	856	* This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively,
	857	* expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */
	858	ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize,
	859	int compressionLevel);
	860
	861	/*! ZSTD_freeCDict() :
	862	* Function frees memory allocated by ZSTD_createCDict().
	863	* If a NULL pointer is passed, no operation is performed. */
	864	ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
	865
	866	/*! ZSTD_compress_usingCDict() :
	867	* Compression using a digested Dictionary.
	868	* Recommended when same dictionary is used multiple times.
	869	* Note : compression level is _decided at dictionary creation time_,
	870	* and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */
	871	ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
	872	void* dst, size_t dstCapacity,
	873	const void* src, size_t srcSize,
	874	const ZSTD_CDict* cdict);
	875
	876
	877	typedef struct ZSTD_DDict_s ZSTD_DDict;
	878
	879	/*! ZSTD_createDDict() :
	880	* Create a digested dictionary, ready to start decompression operation without startup delay.
	881	* dictBuffer can be released after DDict creation, as its content is copied inside DDict. */
	882	ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
	883
	884	/*! ZSTD_freeDDict() :
	885	* Function frees memory allocated with ZSTD_createDDict()
	886	* If a NULL pointer is passed, no operation is performed. */
	887	ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
	888
	889	/*! ZSTD_decompress_usingDDict() :
	890	* Decompression using a digested Dictionary.
	891	* Recommended when same dictionary is used multiple times. */
	892	ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
	893	void* dst, size_t dstCapacity,
	894	const void* src, size_t srcSize,
	895	const ZSTD_DDict* ddict);
	896
	897
	898	/********************************
	899	* Dictionary helper functions
	900	*******************************/
	901
	902	/*! ZSTD_getDictID_fromDict() : Requires v1.4.0+
	903	* Provides the dictID stored within dictionary.
	904	* if @return == 0, the dictionary is not conformant with Zstandard specification.
	905	* It can still be loaded, but as a content-only dictionary. */
	906	ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
	907
	908	/*! ZSTD_getDictID_fromCDict() : Requires v1.5.0+
	909	* Provides the dictID of the dictionary loaded into `cdict`.
	910	* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
	911	* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
	912	ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict);
	913
	914	/*! ZSTD_getDictID_fromDDict() : Requires v1.4.0+
	915	* Provides the dictID of the dictionary loaded into `ddict`.
	916	* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
	917	* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
	918	ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
	919
	920	/*! ZSTD_getDictID_fromFrame() : Requires v1.4.0+
	921	* Provides the dictID required to decompressed the frame stored within `src`.
	922	* If @return == 0, the dictID could not be decoded.
	923	* This could for one of the following reasons :
	924	* - The frame does not require a dictionary to be decoded (most common case).
	925	* - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
	926	* Note : this use case also happens when using a non-conformant dictionary.
	927	* - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
	928	* - This is not a Zstandard frame.
	929	* When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */
	930	ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
	931
	932
	933	/*******************************************************************************
	934	* Advanced dictionary and prefix API (Requires v1.4.0+)
	935	*
	936	* This API allows dictionaries to be used with ZSTD_compress2(),
	937	* ZSTD_compressStream2(), and ZSTD_decompressDCtx(). Dictionaries are sticky, and
	938	* only reset with the context is reset with ZSTD_reset_parameters or
	939	* ZSTD_reset_session_and_parameters. Prefixes are single-use.
	940	******************************************************************************/
	941
	942
	943	/*! ZSTD_CCtx_loadDictionary() : Requires v1.4.0+
	944	* Create an internal CDict from `dict` buffer.
	945	* Decompression will have to use same dictionary.
	946	* @result : 0, or an error code (which can be tested with ZSTD_isError()).
	947	* Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,
	948	* meaning "return to no-dictionary mode".
	949	* Note 1 : Dictionary is sticky, it will be used for all future compressed frames.
	950	* To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters).
	951	* Note 2 : Loading a dictionary involves building tables.
	952	* It's also a CPU consuming operation, with non-negligible impact on latency.
	953	* Tables are dependent on compression parameters, and for this reason,
	954	* compression parameters can no longer be changed after loading a dictionary.
	955	* Note 3 :`dict` content will be copied internally.
	956	* Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead.
	957	* In such a case, dictionary buffer must outlive its users.
	958	* Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
	959	* to precisely select how dictionary content must be interpreted. */
	960	ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
	961
	962	/*! ZSTD_CCtx_refCDict() : Requires v1.4.0+
	963	* Reference a prepared dictionary, to be used for all next compressed frames.
	964	* Note that compression parameters are enforced from within CDict,
	965	* and supersede any compression parameter previously set within CCtx.
	966	* The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
	967	* The ignored parameters will be used again if the CCtx is returned to no-dictionary mode.
	968	* The dictionary will remain valid for future compressed frames using same CCtx.
	969	* @result : 0, or an error code (which can be tested with ZSTD_isError()).
	970	* Special : Referencing a NULL CDict means "return to no-dictionary mode".
	971	* Note 1 : Currently, only one dictionary can be managed.
	972	* Referencing a new dictionary effectively "discards" any previous one.
	973	* Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */
	974	ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
	975
	976	/*! ZSTD_CCtx_refPrefix() : Requires v1.4.0+
	977	* Reference a prefix (single-usage dictionary) for next compressed frame.
	978	* A prefix is only used once. Tables are discarded at end of frame (ZSTD_e_end).
	979	* Decompression will need same prefix to properly regenerate data.
	980	* Compressing with a prefix is similar in outcome as performing a diff and compressing it,
	981	* but performs much faster, especially during decompression (compression speed is tunable with compression level).
	982	* @result : 0, or an error code (which can be tested with ZSTD_isError()).
	983	* Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
	984	* Note 1 : Prefix buffer is referenced. It must outlive compression.
	985	* Its content must remain unmodified during compression.
	986	* Note 2 : If the intention is to diff some large src data blob with some prior version of itself,
	987	* ensure that the window size is large enough to contain the entire source.
	988	* See ZSTD_c_windowLog.
	989	* Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
	990	* It's a CPU consuming operation, with non-negligible impact on latency.
	991	* If there is a need to use the same prefix multiple times, consider loadDictionary instead.
	992	* Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
	993	* Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
	994	ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
	995	const void* prefix, size_t prefixSize);
	996
	997	/*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+
	998	* Create an internal DDict from dict buffer,
	999	* to be used to decompress next frames.
	1000	* The dictionary remains valid for all future frames, until explicitly invalidated.
	1001	* @result : 0, or an error code (which can be tested with ZSTD_isError()).
	1002	* Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
	1003	* meaning "return to no-dictionary mode".
	1004	* Note 1 : Loading a dictionary involves building tables,
	1005	* which has a non-negligible impact on CPU usage and latency.
	1006	* It's recommended to "load once, use many times", to amortize the cost
	1007	* Note 2 :`dict` content will be copied internally, so `dict` can be released after loading.
	1008	* Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead.
	1009	* Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of
	1010	* how dictionary content is loaded and interpreted.
	1011	*/
	1012	ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
	1013
	1014	/*! ZSTD_DCtx_refDDict() : Requires v1.4.0+
	1015	* Reference a prepared dictionary, to be used to decompress next frames.
	1016	* The dictionary remains active for decompression of future frames using same DCtx.
	1017	*
	1018	* If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function
	1019	* will store the DDict references in a table, and the DDict used for decompression
	1020	* will be determined at decompression time, as per the dict ID in the frame.
	1021	* The memory for the table is allocated on the first call to refDDict, and can be
	1022	* freed with ZSTD_freeDCtx().
	1023	*
	1024	* @result : 0, or an error code (which can be tested with ZSTD_isError()).
	1025	* Note 1 : Currently, only one dictionary can be managed.
	1026	* Referencing a new dictionary effectively "discards" any previous one.
	1027	* Special: referencing a NULL DDict means "return to no-dictionary mode".
	1028	* Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
	1029	*/
	1030	ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
	1031
	1032	/*! ZSTD_DCtx_refPrefix() : Requires v1.4.0+
	1033	* Reference a prefix (single-usage dictionary) to decompress next frame.
	1034	* This is the reverse operation of ZSTD_CCtx_refPrefix(),
	1035	* and must use the same prefix as the one used during compression.
	1036	* Prefix is only used once. Reference is discarded at end of frame.
	1037	* End of frame is reached when ZSTD_decompressStream() returns 0.
	1038	* @result : 0, or an error code (which can be tested with ZSTD_isError()).
	1039	* Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary
	1040	* Note 2 : Prefix buffer is referenced. It must outlive decompression.
	1041	* Prefix buffer must remain unmodified up to the end of frame,
	1042	* reached when ZSTD_decompressStream() returns 0.
	1043	* Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
	1044	* Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
	1045	* Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
	1046	* A full dictionary is more costly, as it requires building tables.
	1047	*/
	1048	ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx,
	1049	const void* prefix, size_t prefixSize);
	1050
	1051	/* === Memory management === */
	1052
	1053	/! ZSTD_sizeof_() : Requires v1.4.0+
	1054	* These functions give the _current_ memory usage of selected object.
	1055	* Note that object memory usage can evolve (increase or decrease) over time. */
	1056	ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
	1057	ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
	1058	ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
	1059	ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
	1060	ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
	1061	ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
	1062
	1063	#endif /* ZSTD_H_235446 */
	1064
	1065
	1066	/* **************************************************************************************
	1067	* ADVANCED AND EXPERIMENTAL FUNCTIONS
	1068	****************************************************************************************
	1069	* The definitions in the following section are considered experimental.
	1070	* They are provided for advanced scenarios.
	1071	* They should never be used with a dynamic library, as prototypes may change in the future.
	1072	* Use them only in association with static linking.
	1073	* ***************************************************************************************/
	1074
	1075	#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY)
	1076	#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY
	1077
	1078	/* This can be overridden externally to hide static symbols. */
	1079	#ifndef ZSTDLIB_STATIC_API
	1080	# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
	1081	# define ZSTDLIB_STATIC_API __declspec(dllexport) ZSTDLIB_VISIBLE
	1082	# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
	1083	# define ZSTDLIB_STATIC_API __declspec(dllimport) ZSTDLIB_VISIBLE
	1084	# else
	1085	# define ZSTDLIB_STATIC_API ZSTDLIB_VISIBLE
	1086	# endif
	1087	#endif
	1088
	1089	/* Deprecation warnings :
	1090	* Should these warnings be a problem, it is generally possible to disable them,
	1091	* typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual.
	1092	* Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS.
	1093	*/
	1094	#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS
	1095	# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API /* disable deprecation warnings */
	1096	#else
	1097	# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
	1098	# define ZSTD_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_STATIC_API
	1099	# elif (defined(GNUC) && (GNUC > 4 \|\| (GNUC == 4 && GNUC_MINOR >= 5))) \|\| defined(__clang__)
	1100	# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __attribute__((deprecated(message)))
	1101	# elif defined(__GNUC__) && (__GNUC__ >= 3)
	1102	# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __attribute__((deprecated))
	1103	# elif defined(_MSC_VER)
	1104	# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __declspec(deprecated(message))
	1105	# else
	1106	# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler")
	1107	# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API
	1108	# endif
	1109	#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */
	1110
	1111	/****************************************************************************************
	1112	* experimental API (static linking only)
	1113	****************************************************************************************
	1114	* The following symbols and constants
	1115	* are not planned to join "stable API" status in the near future.
	1116	* They can still change in future versions.
	1117	* Some of them are planned to remain in the static_only section indefinitely.
	1118	* Some of them might be removed in the future (especially when redundant with existing stable functions)
	1119	* ***************************************************************************************/
	1120
	1121	#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */
	1122	#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2)
	1123	#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */
	1124	#define ZSTD_SKIPPABLEHEADERSIZE 8
	1125
	1126	/* compression parameter bounds */
	1127	#define ZSTD_WINDOWLOG_MAX_32 30
	1128	#define ZSTD_WINDOWLOG_MAX_64 31
	1129	#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
	1130	#define ZSTD_WINDOWLOG_MIN 10
	1131	#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30)
	1132	#define ZSTD_HASHLOG_MIN 6
	1133	#define ZSTD_CHAINLOG_MAX_32 29
	1134	#define ZSTD_CHAINLOG_MAX_64 30
	1135	#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64))
	1136	#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN
	1137	#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
	1138	#define ZSTD_SEARCHLOG_MIN 1
	1139	#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */
	1140	#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */
	1141	#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX
	1142	#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */
	1143	#define ZSTD_STRATEGY_MIN ZSTD_fast
	1144	#define ZSTD_STRATEGY_MAX ZSTD_btultra2
	1145
	1146
	1147	#define ZSTD_OVERLAPLOG_MIN 0
	1148	#define ZSTD_OVERLAPLOG_MAX 9
	1149
	1150	#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame
	1151	* requiring larger than (1<<ZSTD_WINDOWLOG_LIMIT_DEFAULT) window size,
	1152	* to preserve host's memory from unreasonable requirements.
	1153	* This limit can be overridden using ZSTD_DCtx_setParameter(,ZSTD_d_windowLogMax,).
	1154	* The limit does not apply for one-pass decoders (such as ZSTD_decompress()), since no additional memory is allocated */
	1155
	1156
	1157	/* LDM parameter bounds */
	1158	#define ZSTD_LDM_HASHLOG_MIN ZSTD_HASHLOG_MIN
	1159	#define ZSTD_LDM_HASHLOG_MAX ZSTD_HASHLOG_MAX
	1160	#define ZSTD_LDM_MINMATCH_MIN 4
	1161	#define ZSTD_LDM_MINMATCH_MAX 4096
	1162	#define ZSTD_LDM_BUCKETSIZELOG_MIN 1
	1163	#define ZSTD_LDM_BUCKETSIZELOG_MAX 8
	1164	#define ZSTD_LDM_HASHRATELOG_MIN 0
	1165	#define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN)
	1166
	1167	/* Advanced parameter bounds */
	1168	#define ZSTD_TARGETCBLOCKSIZE_MIN 64
	1169	#define ZSTD_TARGETCBLOCKSIZE_MAX ZSTD_BLOCKSIZE_MAX
	1170	#define ZSTD_SRCSIZEHINT_MIN 0
	1171	#define ZSTD_SRCSIZEHINT_MAX INT_MAX
	1172
	1173
	1174	/* --- Advanced types --- */
	1175
	1176	typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params;
	1177
	1178	typedef struct {
	1179	unsigned int offset; /* The offset of the match. (NOT the same as the offset code)
	1180	* If offset == 0 and matchLength == 0, this sequence represents the last
	1181	* literals in the block of litLength size.
	1182	*/
	1183
	1184	unsigned int litLength; /* Literal length of the sequence. */
	1185	unsigned int matchLength; /* Match length of the sequence. */
	1186
	1187	/* Note: Users of this API may provide a sequence with matchLength == litLength == offset == 0.
	1188	* In this case, we will treat the sequence as a marker for a block boundary.
	1189	*/
	1190
	1191	unsigned int rep; /* Represents which repeat offset is represented by the field 'offset'.
	1192	* Ranges from [0, 3].
	1193	*
	1194	* Repeat offsets are essentially previous offsets from previous sequences sorted in
	1195	* recency order. For more detail, see doc/zstd_compression_format.md
	1196	*
	1197	* If rep == 0, then 'offset' does not contain a repeat offset.
	1198	* If rep > 0:
	1199	* If litLength != 0:
	1200	* rep == 1 --> offset == repeat_offset_1
	1201	* rep == 2 --> offset == repeat_offset_2
	1202	* rep == 3 --> offset == repeat_offset_3
	1203	* If litLength == 0:
	1204	* rep == 1 --> offset == repeat_offset_2
	1205	* rep == 2 --> offset == repeat_offset_3
	1206	* rep == 3 --> offset == repeat_offset_1 - 1
	1207	*
	1208	* Note: This field is optional. ZSTD_generateSequences() will calculate the value of
	1209	* 'rep', but repeat offsets do not necessarily need to be calculated from an external
	1210	* sequence provider's perspective. For example, ZSTD_compressSequences() does not
	1211	* use this 'rep' field at all (as of now).
	1212	*/
	1213	} ZSTD_Sequence;
	1214
	1215	typedef struct {
	1216	unsigned windowLog; /*< largest match distance : larger == more compression, more memory needed during decompression /
	1217	unsigned chainLog; /*< fully searched segment : larger == more compression, slower, more memory (useless for fast) /
	1218	unsigned hashLog; /*< dispatch table : larger == faster, more memory /
	1219	unsigned searchLog; /*< nb of searches : larger == more compression, slower /
	1220	unsigned minMatch; /*< match length searched : larger == faster decompression, sometimes less compression /
	1221	unsigned targetLength; /*< acceptable match size for optimal parser (only) : larger == more compression, slower /
	1222	ZSTD_strategy strategy; /*< see ZSTD_strategy definition above /
	1223	} ZSTD_compressionParameters;
	1224
	1225	typedef struct {
	1226	int contentSizeFlag; /*< 1: content size will be in frame header (when known) /
	1227	int checksumFlag; /*< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection /
	1228	int noDictIDFlag; /*< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) /
	1229	} ZSTD_frameParameters;
	1230
	1231	typedef struct {
	1232	ZSTD_compressionParameters cParams;
	1233	ZSTD_frameParameters fParams;
	1234	} ZSTD_parameters;
	1235
	1236	typedef enum {
	1237	ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */
	1238	ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */
	1239	ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */
	1240	} ZSTD_dictContentType_e;
	1241
	1242	typedef enum {
	1243	ZSTD_dlm_byCopy = 0, /*< Copy dictionary content internally /
	1244	ZSTD_dlm_byRef = 1 /*< Reference dictionary content -- the dictionary buffer must outlive its users. /
	1245	} ZSTD_dictLoadMethod_e;
	1246
	1247	typedef enum {
	1248	ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */
	1249	ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number.
	1250	* Useful to save 4 bytes per generated frame.
	1251	* Decoder cannot recognise automatically this format, requiring this instruction. */
	1252	} ZSTD_format_e;
	1253
	1254	typedef enum {
	1255	/* Note: this enum controls ZSTD_d_forceIgnoreChecksum */
	1256	ZSTD_d_validateChecksum = 0,
	1257	ZSTD_d_ignoreChecksum = 1
	1258	} ZSTD_forceIgnoreChecksum_e;
	1259
	1260	typedef enum {
	1261	/* Note: this enum controls ZSTD_d_refMultipleDDicts */
	1262	ZSTD_rmd_refSingleDDict = 0,
	1263	ZSTD_rmd_refMultipleDDicts = 1
	1264	} ZSTD_refMultipleDDicts_e;
	1265
	1266	typedef enum {
	1267	/* Note: this enum and the behavior it controls are effectively internal
	1268	* implementation details of the compressor. They are expected to continue
	1269	* to evolve and should be considered only in the context of extremely
	1270	* advanced performance tuning.
	1271	*
	1272	* Zstd currently supports the use of a CDict in three ways:
	1273	*
	1274	* - The contents of the CDict can be copied into the working context. This
	1275	* means that the compression can search both the dictionary and input
	1276	* while operating on a single set of internal tables. This makes
	1277	* the compression faster per-byte of input. However, the initial copy of
	1278	* the CDict's tables incurs a fixed cost at the beginning of the
	1279	* compression. For small compressions (< 8 KB), that copy can dominate
	1280	* the cost of the compression.
	1281	*
	1282	* - The CDict's tables can be used in-place. In this model, compression is
	1283	* slower per input byte, because the compressor has to search two sets of
	1284	* tables. However, this model incurs no start-up cost (as long as the
	1285	* working context's tables can be reused). For small inputs, this can be
	1286	* faster than copying the CDict's tables.
	1287	*
	1288	* - The CDict's tables are not used at all, and instead we use the working
	1289	* context alone to reload the dictionary and use params based on the source
	1290	* size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict().
	1291	* This method is effective when the dictionary sizes are very small relative
	1292	* to the input size, and the input size is fairly large to begin with.
	1293	*
	1294	* Zstd has a simple internal heuristic that selects which strategy to use
	1295	* at the beginning of a compression. However, if experimentation shows that
	1296	* Zstd is making poor choices, it is possible to override that choice with
	1297	* this enum.
	1298	*/
	1299	ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */
	1300	ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */
	1301	ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */
	1302	ZSTD_dictForceLoad = 3 /* Always reload the dictionary */
	1303	} ZSTD_dictAttachPref_e;
	1304
	1305	typedef enum {
	1306	ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level.
	1307	* Negative compression levels will be uncompressed, and positive compression
	1308	* levels will be compressed. */
	1309	ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be
	1310	* emitted if Huffman compression is not profitable. */
	1311	ZSTD_lcm_uncompressed = 2 /*< Always emit uncompressed literals. /
	1312	} ZSTD_literalCompressionMode_e;
	1313
	1314	typedef enum {
	1315	/* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final
	1316	* decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable
	1317	* or ZSTD_ps_disable allow for a force enable/disable the feature.
	1318	*/
	1319	ZSTD_ps_auto = 0, /* Let the library automatically determine whether the feature shall be enabled */
	1320	ZSTD_ps_enable = 1, /* Force-enable the feature */
	1321	ZSTD_ps_disable = 2 /* Do not use the feature */
	1322	} ZSTD_paramSwitch_e;
	1323
	1324	/***************************************
	1325	* Frame size functions
	1326	***************************************/
	1327
	1328	/*! ZSTD_findDecompressedSize() :
	1329	* `src` should point to the start of a series of ZSTD encoded and/or skippable frames
	1330	* `srcSize` must be the _exact_ size of this series
	1331	* (i.e. there should be a frame boundary at `src + srcSize`)
	1332	* @return : - decompressed size of all data in all successive frames
	1333	* - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
	1334	* - if an error occurred: ZSTD_CONTENTSIZE_ERROR
	1335	*
	1336	* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
	1337	* When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
	1338	* In which case, it's necessary to use streaming mode to decompress data.
	1339	* note 2 : decompressed size is always present when compression is done with ZSTD_compress()
	1340	* note 3 : decompressed size can be very large (64-bits value),
	1341	* potentially larger than what local system can handle as a single memory segment.
	1342	* In which case, it's necessary to use streaming mode to decompress data.
	1343	* note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
	1344	* Always ensure result fits within application's authorized limits.
	1345	* Each application can set its own limits.
	1346	* note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
	1347	* read each contained frame header. This is fast as most of the data is skipped,
	1348	* however it does mean that all frame data must be present and valid. */
	1349	ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
	1350
	1351	/*! ZSTD_decompressBound() :
	1352	* `src` should point to the start of a series of ZSTD encoded and/or skippable frames
	1353	* `srcSize` must be the _exact_ size of this series
	1354	* (i.e. there should be a frame boundary at `src + srcSize`)
	1355	* @return : - upper-bound for the decompressed size of all data in all successive frames
	1356	* - if an error occurred: ZSTD_CONTENTSIZE_ERROR
	1357	*
	1358	* note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame.
	1359	* note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`.
	1360	* in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value.
	1361	* note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by:
	1362	* upper-bound = # blocks * min(128 KB, Window_Size)
	1363	*/
	1364	ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize);
	1365
	1366	/*! ZSTD_frameHeaderSize() :
	1367	* srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
	1368	* @return : size of the Frame Header,
	1369	* or an error code (if srcSize is too small) */
	1370	ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
	1371
	1372	typedef enum {
	1373	ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */
	1374	ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */
	1375	} ZSTD_sequenceFormat_e;
	1376
	1377	/*! ZSTD_generateSequences() :
	1378	* Generate sequences using ZSTD_compress2, given a source buffer.
	1379	*
	1380	* Each block will end with a dummy sequence
	1381	* with offset == 0, matchLength == 0, and litLength == length of last literals.
	1382	* litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0)
	1383	* simply acts as a block delimiter.
	1384	*
	1385	* zc can be used to insert custom compression params.
	1386	* This function invokes ZSTD_compress2
	1387	*
	1388	* The output of this function can be fed into ZSTD_compressSequences() with CCtx
	1389	* setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters
	1390	* @return : number of sequences generated
	1391	*/
	1392
	1393	ZSTDLIB_STATIC_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
	1394	size_t outSeqsSize, const void* src, size_t srcSize);
	1395
	1396	/*! ZSTD_mergeBlockDelimiters() :
	1397	* Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals
	1398	* by merging them into into the literals of the next sequence.
	1399	*
	1400	* As such, the final generated result has no explicit representation of block boundaries,
	1401	* and the final last literals segment is not represented in the sequences.
	1402	*
	1403	* The output of this function can be fed into ZSTD_compressSequences() with CCtx
	1404	* setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters
	1405	* @return : number of sequences left after merging
	1406	*/
	1407	ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize);
	1408
	1409	/*! ZSTD_compressSequences() :
	1410	* Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst.
	1411	* If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
	1412	* The entire source is compressed into a single frame.
	1413	*
	1414	* The compression behavior changes based on cctx params. In particular:
	1415	* If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain
	1416	* no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on
	1417	* the block size derived from the cctx, and sequences may be split. This is the default setting.
	1418	*
	1419	* If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain
	1420	* block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided.
	1421	*
	1422	* If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined
	1423	* behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for
	1424	* specifics regarding offset/matchlength requirements) then the function will bail out and return an error.
	1425	*
	1426	* In addition to the two adjustable experimental params, there are other important cctx params.
	1427	* - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN.
	1428	* - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression.
	1429	* - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset
	1430	* is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md
	1431	*
	1432	* Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
	1433	* Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
	1434	* and cannot emit an RLE block that disagrees with the repcode history
	1435	* @return : final compressed size or a ZSTD error.
	1436	*/
	1437	ZSTDLIB_STATIC_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize,
	1438	const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
	1439	const void* src, size_t srcSize);
	1440
	1441
	1442	/*! ZSTD_writeSkippableFrame() :
	1443	* Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
	1444	*
	1445	* Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number,
	1446	* ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
	1447	* As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
	1448	* the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
	1449	*
	1450	* Returns an error if destination buffer is not large enough, if the source size is not representable
	1451	* with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid).
	1452	*
	1453	* @return : number of bytes written or a ZSTD error.
	1454	*/
	1455	ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity,
	1456	const void* src, size_t srcSize, unsigned magicVariant);
	1457
	1458	/*! ZSTD_readSkippableFrame() :
	1459	* Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
	1460	*
	1461	* The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
	1462	* i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested
	1463	* in the magicVariant.
	1464	*
	1465	* Returns an error if destination buffer is not large enough, or if the frame is not skippable.
	1466	*
	1467	* @return : number of bytes written or a ZSTD error.
	1468	*/
	1469	ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
	1470	const void* src, size_t srcSize);
	1471
	1472	/*! ZSTD_isSkippableFrame() :
	1473	* Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame.
	1474	*/
	1475	ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
	1476
	1477
	1478
	1479	/***************************************
	1480	* Memory management
	1481	***************************************/
	1482
	1483	/! ZSTD_estimate() :
	1484	* These functions make it possible to estimate memory usage
	1485	* of a future {D,C}Ctx, before its creation.
	1486	*
	1487	* ZSTD_estimateCCtxSize() will provide a memory budget large enough
	1488	* for any compression level up to selected one.
	1489	* Note : Unlike ZSTD_estimateCStreamSize*(), this estimate
	1490	* does not include space for a window buffer.
	1491	* Therefore, the estimation is only guaranteed for single-shot compressions, not streaming.
	1492	* The estimate will assume the input may be arbitrarily large,
	1493	* which is the worst case.
	1494	*
	1495	* When srcSize can be bound by a known and rather "small" value,
	1496	* this fact can be used to provide a tighter estimation
	1497	* because the CCtx compression context will need less memory.
	1498	* This tighter estimation can be provided by more advanced functions
	1499	* ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(),
	1500	* and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().
	1501	* Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.
	1502	*
	1503	* Note 2 : only single-threaded compression is supported.
	1504	* ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
	1505	*/
	1506	ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
	1507	ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
	1508	ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params);
	1509	ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
	1510
	1511	/*! ZSTD_estimateCStreamSize() :
	1512	* ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.
	1513	* It will also consider src size to be arbitrarily "large", which is worst case.
	1514	* If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation.
	1515	* ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
	1516	* ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1.
	1517	* Note : CStream size estimation is only correct for single-threaded compression.
	1518	* ZSTD_DStream memory budget depends on window Size.
	1519	* This information can be passed manually, using ZSTD_estimateDStreamSize,
	1520	* or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
	1521	* Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
	1522	* an internal ?Dict will be created, which additional size is not estimated here.
	1523	* In this case, get total size by adding ZSTD_estimate?DictSize */
	1524	ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
	1525	ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
	1526	ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
	1527	ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t windowSize);
	1528	ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
	1529
	1530	/*! ZSTD_estimate?DictSize() :
	1531	* ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
	1532	* ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced().
	1533	* Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller.
	1534	*/
	1535	ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
	1536	ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod);
	1537	ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod);
	1538
	1539	/! ZSTD_initStatic() :
	1540	* Initialize an object using a pre-allocated fixed-size buffer.
	1541	* workspace: The memory area to emplace the object into.
	1542	* Provided pointer must be 8-bytes aligned.
	1543	* Buffer must outlive object.
	1544	* workspaceSize: Use ZSTD_estimate*Size() to determine
	1545	* how large workspace must be to support target scenario.
	1546	* @return : pointer to object (same address as workspace, just different type),
	1547	* or NULL if error (size too small, incorrect alignment, etc.)
	1548	* Note : zstd will never resize nor malloc() when using a static buffer.
	1549	* If the object requires more memory than available,
	1550	* zstd will just error out (typically ZSTD_error_memory_allocation).
	1551	* Note 2 : there is no corresponding "free" function.
	1552	* Since workspace is allocated externally, it must be freed externally too.
	1553	* Note 3 : cParams : use ZSTD_getCParams() to convert a compression level
	1554	* into its associated cParams.
	1555	* Limitation 1 : currently not compatible with internal dictionary creation, triggered by
	1556	* ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict().
	1557	* Limitation 2 : static cctx currently not compatible with multi-threading.
	1558	* Limitation 3 : static dctx is incompatible with legacy support.
	1559	*/
	1560	ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
	1561	ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /*< same as ZSTD_initStaticCCtx() /
	1562
	1563	ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize);
	1564	ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /*< same as ZSTD_initStaticDCtx() /
	1565
	1566	ZSTDLIB_STATIC_API const ZSTD_CDict* ZSTD_initStaticCDict(
	1567	void* workspace, size_t workspaceSize,
	1568	const void* dict, size_t dictSize,
	1569	ZSTD_dictLoadMethod_e dictLoadMethod,
	1570	ZSTD_dictContentType_e dictContentType,
	1571	ZSTD_compressionParameters cParams);
	1572
	1573	ZSTDLIB_STATIC_API const ZSTD_DDict* ZSTD_initStaticDDict(
	1574	void* workspace, size_t workspaceSize,
	1575	const void* dict, size_t dictSize,
	1576	ZSTD_dictLoadMethod_e dictLoadMethod,
	1577	ZSTD_dictContentType_e dictContentType);
	1578
	1579
	1580	/*! Custom memory allocation :
	1581	* These prototypes make it possible to pass your own allocation/free functions.
	1582	* ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below.
	1583	* All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones.
	1584	*/
	1585	typedef void* (ZSTD_allocFunction) (void opaque, size_t size);
	1586	typedef void (ZSTD_freeFunction) (void opaque, void* address);
	1587	typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
	1588	static
	1589	#ifdef __GNUC__
	1590	__attribute__((__unused__))
	1591	#endif
	1592	ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /*< this constant defers to stdlib's functions /
	1593
	1594	ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
	1595	ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
	1596	ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
	1597	ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
	1598
	1599	ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
	1600	ZSTD_dictLoadMethod_e dictLoadMethod,
	1601	ZSTD_dictContentType_e dictContentType,
	1602	ZSTD_compressionParameters cParams,
	1603	ZSTD_customMem customMem);
	1604
	1605	/*! Thread pool :
	1606	* These prototypes make it possible to share a thread pool among multiple compression contexts.
	1607	* This can limit resources for applications with multiple threads where each one uses
	1608	* a threaded compression mode (via ZSTD_c_nbWorkers parameter).
	1609	* ZSTD_createThreadPool creates a new thread pool with a given number of threads.
	1610	* Note that the lifetime of such pool must exist while being used.
	1611	* ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value
	1612	* to use an internal thread pool).
	1613	* ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer.
	1614	*/
	1615	typedef struct POOL_ctx_s ZSTD_threadPool;
	1616	ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads);
	1617	ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); /* accept NULL pointer */
	1618	ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool);
	1619
	1620
	1621	/*
	1622	* This API is temporary and is expected to change or disappear in the future!
	1623	*/
	1624	ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced2(
	1625	const void* dict, size_t dictSize,
	1626	ZSTD_dictLoadMethod_e dictLoadMethod,
	1627	ZSTD_dictContentType_e dictContentType,
	1628	const ZSTD_CCtx_params* cctxParams,
	1629	ZSTD_customMem customMem);
	1630
	1631	ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_advanced(
	1632	const void* dict, size_t dictSize,
	1633	ZSTD_dictLoadMethod_e dictLoadMethod,
	1634	ZSTD_dictContentType_e dictContentType,
	1635	ZSTD_customMem customMem);
	1636
	1637
	1638	/***************************************
	1639	* Advanced compression functions
	1640	***************************************/
	1641
	1642	/*! ZSTD_createCDict_byReference() :
	1643	* Create a digested dictionary for compression
	1644	* Dictionary content is just referenced, not duplicated.
	1645	* As a consequence, `dictBuffer` must outlive CDict,
	1646	* and its content must remain unmodified throughout the lifetime of CDict.
	1647	* note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */
	1648	ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
	1649
	1650	/*! ZSTD_getCParams() :
	1651	* @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
	1652	* `estimatedSrcSize` value is optional, select 0 if not known */
	1653	ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
	1654
	1655	/*! ZSTD_getParams() :
	1656	* same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`.
	1657	* All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */
	1658	ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
	1659
	1660	/*! ZSTD_checkCParams() :
	1661	* Ensure param values remain within authorized range.
	1662	* @return 0 on success, or an error code (can be checked with ZSTD_isError()) */
	1663	ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
	1664
	1665	/*! ZSTD_adjustCParams() :
	1666	* optimize params for a given `srcSize` and `dictSize`.
	1667	* `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN.
	1668	* `dictSize` must be `0` when there is no dictionary.
	1669	* cPar can be invalid : all parameters will be clamped within valid range in the @return struct.
	1670	* This function never fails (wide contract) */
	1671	ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
	1672
	1673	/*! ZSTD_compress_advanced() :
	1674	* Note : this function is now DEPRECATED.
	1675	* It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
	1676	* This prototype will generate compilation warnings. */
	1677	ZSTD_DEPRECATED("use ZSTD_compress2")
	1678	size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
	1679	void* dst, size_t dstCapacity,
	1680	const void* src, size_t srcSize,
	1681	const void* dict,size_t dictSize,
	1682	ZSTD_parameters params);
	1683
	1684	/*! ZSTD_compress_usingCDict_advanced() :
	1685	* Note : this function is now DEPRECATED.
	1686	* It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
	1687	* This prototype will generate compilation warnings. */
	1688	ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary")
	1689	size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
	1690	void* dst, size_t dstCapacity,
	1691	const void* src, size_t srcSize,
	1692	const ZSTD_CDict* cdict,
	1693	ZSTD_frameParameters fParams);
	1694
	1695
	1696	/*! ZSTD_CCtx_loadDictionary_byReference() :
	1697	* Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx.
	1698	* It saves some memory, but also requires that `dict` outlives its usage within `cctx` */
	1699	ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
	1700
	1701	/*! ZSTD_CCtx_loadDictionary_advanced() :
	1702	* Same as ZSTD_CCtx_loadDictionary(), but gives finer control over
	1703	* how to load the dictionary (by copy ? by reference ?)
	1704	* and how to interpret it (automatic ? force raw mode ? full mode only ?) */
	1705	ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
	1706
	1707	/*! ZSTD_CCtx_refPrefix_advanced() :
	1708	* Same as ZSTD_CCtx_refPrefix(), but gives finer control over
	1709	* how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
	1710	ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
	1711
	1712	/* === experimental parameters === */
	1713	/* these parameters can be used with ZSTD_setParameter()
	1714	* they are not guaranteed to remain supported in the future */
	1715
	1716	/* Enables rsyncable mode,
	1717	* which makes compressed files more rsync friendly
	1718	* by adding periodic synchronization points to the compressed data.
	1719	* The target average block size is ZSTD_c_jobSize / 2.
	1720	* It's possible to modify the job size to increase or decrease
	1721	* the granularity of the synchronization point.
	1722	* Once the jobSize is smaller than the window size,
	1723	* it will result in compression ratio degradation.
	1724	* NOTE 1: rsyncable mode only works when multithreading is enabled.
	1725	* NOTE 2: rsyncable performs poorly in combination with long range mode,
	1726	* since it will decrease the effectiveness of synchronization points,
	1727	* though mileage may vary.
	1728	* NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s.
	1729	* If the selected compression level is already running significantly slower,
	1730	* the overall speed won't be significantly impacted.
	1731	*/
	1732	#define ZSTD_c_rsyncable ZSTD_c_experimentalParam1
	1733
	1734	/* Select a compression format.
	1735	* The value must be of type ZSTD_format_e.
	1736	* See ZSTD_format_e enum definition for details */
	1737	#define ZSTD_c_format ZSTD_c_experimentalParam2
	1738
	1739	/* Force back-reference distances to remain < windowSize,
	1740	* even when referencing into Dictionary content (default:0) */
	1741	#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3
	1742
	1743	/* Controls whether the contents of a CDict
	1744	* are used in place, or copied into the working context.
	1745	* Accepts values from the ZSTD_dictAttachPref_e enum.
	1746	* See the comments on that enum for an explanation of the feature. */
	1747	#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4
	1748
	1749	/* Controlled with ZSTD_paramSwitch_e enum.
	1750	* Default is ZSTD_ps_auto.
	1751	* Set to ZSTD_ps_disable to never compress literals.
	1752	* Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals
	1753	* may still be emitted if huffman is not beneficial to use.)
	1754	*
	1755	* By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
	1756	* literals compression based on the compression parameters - specifically,
	1757	* negative compression levels do not use literal compression.
	1758	*/
	1759	#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5
	1760
	1761	/* Tries to fit compressed block size to be around targetCBlockSize.
	1762	* No target when targetCBlockSize == 0.
	1763	* There is no guarantee on compressed block size (default:0) */
	1764	#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6
	1765
	1766	/* User's best guess of source size.
	1767	* Hint is not valid when srcSizeHint == 0.
	1768	* There is no guarantee that hint is close to actual source size,
	1769	* but compression ratio may regress significantly if guess considerably underestimates */
	1770	#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7
	1771
	1772	/* Controls whether the new and experimental "dedicated dictionary search
	1773	* structure" can be used. This feature is still rough around the edges, be
	1774	* prepared for surprising behavior!
	1775	*
	1776	* How to use it:
	1777	*
	1778	* When using a CDict, whether to use this feature or not is controlled at
	1779	* CDict creation, and it must be set in a CCtxParams set passed into that
	1780	* construction (via ZSTD_createCDict_advanced2()). A compression will then
	1781	* use the feature or not based on how the CDict was constructed; the value of
	1782	* this param, set in the CCtx, will have no effect.
	1783	*
	1784	* However, when a dictionary buffer is passed into a CCtx, such as via
	1785	* ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control
	1786	* whether the CDict that is created internally can use the feature or not.
	1787	*
	1788	* What it does:
	1789	*
	1790	* Normally, the internal data structures of the CDict are analogous to what
	1791	* would be stored in a CCtx after compressing the contents of a dictionary.
	1792	* To an approximation, a compression using a dictionary can then use those
	1793	* data structures to simply continue what is effectively a streaming
	1794	* compression where the simulated compression of the dictionary left off.
	1795	* Which is to say, the search structures in the CDict are normally the same
	1796	* format as in the CCtx.
	1797	*
	1798	* It is possible to do better, since the CDict is not like a CCtx: the search
	1799	* structures are written once during CDict creation, and then are only read
	1800	* after that, while the search structures in the CCtx are both read and
	1801	* written as the compression goes along. This means we can choose a search
	1802	* structure for the dictionary that is read-optimized.
	1803	*
	1804	* This feature enables the use of that different structure.
	1805	*
	1806	* Note that some of the members of the ZSTD_compressionParameters struct have
	1807	* different semantics and constraints in the dedicated search structure. It is
	1808	* highly recommended that you simply set a compression level in the CCtxParams
	1809	* you pass into the CDict creation call, and avoid messing with the cParams
	1810	* directly.
	1811	*
	1812	* Effects:
	1813	*
	1814	* This will only have any effect when the selected ZSTD_strategy
	1815	* implementation supports this feature. Currently, that's limited to
	1816	* ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2.
	1817	*
	1818	* Note that this means that the CDict tables can no longer be copied into the
	1819	* CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be
	1820	* usable. The dictionary can only be attached or reloaded.
	1821	*
	1822	* In general, you should expect compression to be faster--sometimes very much
	1823	* so--and CDict creation to be slightly slower. Eventually, we will probably
	1824	* make this mode the default.
	1825	*/
	1826	#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8
	1827
	1828	/* ZSTD_c_stableInBuffer
	1829	* Experimental parameter.
	1830	* Default is 0 == disabled. Set to 1 to enable.
	1831	*
	1832	* Tells the compressor that the ZSTD_inBuffer will ALWAYS be the same
	1833	* between calls, except for the modifications that zstd makes to pos (the
	1834	* caller must not modify pos). This is checked by the compressor, and
	1835	* compression will fail if it ever changes. This means the only flush
	1836	* mode that makes sense is ZSTD_e_end, so zstd will error if ZSTD_e_end
	1837	* is not used. The data in the ZSTD_inBuffer in the range [src, src + pos)
	1838	* MUST not be modified during compression or you will get data corruption.
	1839	*
	1840	* When this flag is enabled zstd won't allocate an input window buffer,
	1841	* because the user guarantees it can reference the ZSTD_inBuffer until
	1842	* the frame is complete. But, it will still allocate an output buffer
	1843	* large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also
	1844	* avoid the memcpy() from the input buffer to the input window buffer.
	1845	*
	1846	* NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used.
	1847	* That means this flag cannot be used with ZSTD_compressStream().
	1848	*
	1849	* NOTE: So long as the ZSTD_inBuffer always points to valid memory, using
	1850	* this flag is ALWAYS memory safe, and will never access out-of-bounds
	1851	* memory. However, compression WILL fail if you violate the preconditions.
	1852	*
	1853	* WARNING: The data in the ZSTD_inBuffer in the range [dst, dst + pos) MUST
	1854	* not be modified during compression or you will get data corruption. This
	1855	* is because zstd needs to reference data in the ZSTD_inBuffer to find
	1856	* matches. Normally zstd maintains its own window buffer for this purpose,
	1857	* but passing this flag tells zstd to use the user provided buffer.
	1858	*/
	1859	#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9
	1860
	1861	/* ZSTD_c_stableOutBuffer
	1862	* Experimental parameter.
	1863	* Default is 0 == disabled. Set to 1 to enable.
	1864	*
	1865	* Tells he compressor that the ZSTD_outBuffer will not be resized between
	1866	* calls. Specifically: (out.size - out.pos) will never grow. This gives the
	1867	* compressor the freedom to say: If the compressed data doesn't fit in the
	1868	* output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to
	1869	* always decompress directly into the output buffer, instead of decompressing
	1870	* into an internal buffer and copying to the output buffer.
	1871	*
	1872	* When this flag is enabled zstd won't allocate an output buffer, because
	1873	* it can write directly to the ZSTD_outBuffer. It will still allocate the
	1874	* input window buffer (see ZSTD_c_stableInBuffer).
	1875	*
	1876	* Zstd will check that (out.size - out.pos) never grows and return an error
	1877	* if it does. While not strictly necessary, this should prevent surprises.
	1878	*/
	1879	#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10
	1880
	1881	/* ZSTD_c_blockDelimiters
	1882	* Default is 0 == ZSTD_sf_noBlockDelimiters.
	1883	*
	1884	* For use with sequence compression API: ZSTD_compressSequences().
	1885	*
	1886	* Designates whether or not the given array of ZSTD_Sequence contains block delimiters
	1887	* and last literals, which are defined as sequences with offset == 0 and matchLength == 0.
	1888	* See the definition of ZSTD_Sequence for more specifics.
	1889	*/
	1890	#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11
	1891
	1892	/* ZSTD_c_validateSequences
	1893	* Default is 0 == disabled. Set to 1 to enable sequence validation.
	1894	*
	1895	* For use with sequence compression API: ZSTD_compressSequences().
	1896	* Designates whether or not we validate sequences provided to ZSTD_compressSequences()
	1897	* during function execution.
	1898	*
	1899	* Without validation, providing a sequence that does not conform to the zstd spec will cause
	1900	* undefined behavior, and may produce a corrupted block.
	1901	*
	1902	* With validation enabled, a if sequence is invalid (see doc/zstd_compression_format.md for
	1903	* specifics regarding offset/matchlength requirements) then the function will bail out and
	1904	* return an error.
	1905	*
	1906	*/
	1907	#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12
	1908
	1909	/* ZSTD_c_useBlockSplitter
	1910	* Controlled with ZSTD_paramSwitch_e enum.
	1911	* Default is ZSTD_ps_auto.
	1912	* Set to ZSTD_ps_disable to never use block splitter.
	1913	* Set to ZSTD_ps_enable to always use block splitter.
	1914	*
	1915	* By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
	1916	* block splitting based on the compression parameters.
	1917	*/
	1918	#define ZSTD_c_useBlockSplitter ZSTD_c_experimentalParam13
	1919
	1920	/* ZSTD_c_useRowMatchFinder
	1921	* Controlled with ZSTD_paramSwitch_e enum.
	1922	* Default is ZSTD_ps_auto.
	1923	* Set to ZSTD_ps_disable to never use row-based matchfinder.
	1924	* Set to ZSTD_ps_enable to force usage of row-based matchfinder.
	1925	*
	1926	* By default, in ZSTD_ps_auto, the library will decide at runtime whether to use
	1927	* the row-based matchfinder based on support for SIMD instructions and the window log.
	1928	* Note that this only pertains to compression strategies: greedy, lazy, and lazy2
	1929	*/
	1930	#define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14
	1931
	1932	/* ZSTD_c_deterministicRefPrefix
	1933	* Default is 0 == disabled. Set to 1 to enable.
	1934	*
	1935	* Zstd produces different results for prefix compression when the prefix is
	1936	* directly adjacent to the data about to be compressed vs. when it isn't.
	1937	* This is because zstd detects that the two buffers are contiguous and it can
	1938	* use a more efficient match finding algorithm. However, this produces different
	1939	* results than when the two buffers are non-contiguous. This flag forces zstd
	1940	* to always load the prefix in non-contiguous mode, even if it happens to be
	1941	* adjacent to the data, to guarantee determinism.
	1942	*
	1943	* If you really care about determinism when using a dictionary or prefix,
	1944	* like when doing delta compression, you should select this option. It comes
	1945	* at a speed penalty of about ~2.5% if the dictionary and data happened to be
	1946	* contiguous, and is free if they weren't contiguous. We don't expect that
	1947	* intentionally making the dictionary and data contiguous will be worth the
	1948	* cost to memcpy() the data.
	1949	*/
	1950	#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15
	1951
	1952	/*! ZSTD_CCtx_getParameter() :
	1953	* Get the requested compression parameter value, selected by enum ZSTD_cParameter,
	1954	* and store it into int* value.
	1955	* @return : 0, or an error code (which can be tested with ZSTD_isError()).
	1956	*/
	1957	ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value);
	1958
	1959
	1960	/*! ZSTD_CCtx_params :
	1961	* Quick howto :
	1962	* - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure
	1963	* - ZSTD_CCtxParams_setParameter() : Push parameters one by one into
	1964	* an existing ZSTD_CCtx_params structure.
	1965	* This is similar to
	1966	* ZSTD_CCtx_setParameter().
	1967	* - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to
	1968	* an existing CCtx.
	1969	* These parameters will be applied to
	1970	* all subsequent frames.
	1971	* - ZSTD_compressStream2() : Do compression using the CCtx.
	1972	* - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer.
	1973	*
	1974	* This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams()
	1975	* for static allocation of CCtx for single-threaded compression.
	1976	*/
	1977	ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
	1978	ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); /* accept NULL pointer */
	1979
	1980	/*! ZSTD_CCtxParams_reset() :
	1981	* Reset params to default values.
	1982	*/
	1983	ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params);
	1984
	1985	/*! ZSTD_CCtxParams_init() :
	1986	* Initializes the compression parameters of cctxParams according to
	1987	* compression level. All other parameters are reset to their default values.
	1988	*/
	1989	ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel);
	1990
	1991	/*! ZSTD_CCtxParams_init_advanced() :
	1992	* Initializes the compression and frame parameters of cctxParams according to
	1993	* params. All other parameters are reset to their default values.
	1994	*/
	1995	ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
	1996
	1997	/*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+
	1998	* Similar to ZSTD_CCtx_setParameter.
	1999	* Set one compression parameter, selected by enum ZSTD_cParameter.
	2000	* Parameters must be applied to a ZSTD_CCtx using
	2001	* ZSTD_CCtx_setParametersUsingCCtxParams().
	2002	* @result : a code representing success or failure (which can be tested with
	2003	* ZSTD_isError()).
	2004	*/
	2005	ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value);
	2006
	2007	/*! ZSTD_CCtxParams_getParameter() :
	2008	* Similar to ZSTD_CCtx_getParameter.
	2009	* Get the requested value of one compression parameter, selected by enum ZSTD_cParameter.
	2010	* @result : 0, or an error code (which can be tested with ZSTD_isError()).
	2011	*/
	2012	ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value);
	2013
	2014	/*! ZSTD_CCtx_setParametersUsingCCtxParams() :
	2015	* Apply a set of ZSTD_CCtx_params to the compression context.
	2016	* This can be done even after compression is started,
	2017	* if nbWorkers==0, this will have no impact until a new compression is started.
	2018	* if nbWorkers>=1, new parameters will be picked up at next job,
	2019	* with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated).
	2020	*/
	2021	ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams(
	2022	ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params);
	2023
	2024	/*! ZSTD_compressStream2_simpleArgs() :
	2025	* Same as ZSTD_compressStream2(),
	2026	* but using only integral types as arguments.
	2027	* This variant might be helpful for binders from dynamic languages
	2028	* which have troubles handling structures containing memory pointers.
	2029	*/
	2030	ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs (
	2031	ZSTD_CCtx* cctx,
	2032	void* dst, size_t dstCapacity, size_t* dstPos,
	2033	const void* src, size_t srcSize, size_t* srcPos,
	2034	ZSTD_EndDirective endOp);
	2035
	2036
	2037	/***************************************
	2038	* Advanced decompression functions
	2039	***************************************/
	2040
	2041	/*! ZSTD_isFrame() :
	2042	* Tells if the content of `buffer` starts with a valid Frame Identifier.
	2043	* Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
	2044	* Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
	2045	* Note 3 : Skippable Frame Identifiers are considered valid. */
	2046	ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
	2047
	2048	/*! ZSTD_createDDict_byReference() :
	2049	* Create a digested dictionary, ready to start decompression operation without startup delay.
	2050	* Dictionary content is referenced, and therefore stays in dictBuffer.
	2051	* It is important that dictBuffer outlives DDict,
	2052	* it must remain read accessible throughout the lifetime of DDict */
	2053	ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
	2054
	2055	/*! ZSTD_DCtx_loadDictionary_byReference() :
	2056	* Same as ZSTD_DCtx_loadDictionary(),
	2057	* but references `dict` content instead of copying it into `dctx`.
	2058	* This saves memory if `dict` remains around.,
	2059	* However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */
	2060	ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
	2061
	2062	/*! ZSTD_DCtx_loadDictionary_advanced() :
	2063	* Same as ZSTD_DCtx_loadDictionary(),
	2064	* but gives direct control over
	2065	* how to load the dictionary (by copy ? by reference ?)
	2066	* and how to interpret it (automatic ? force raw mode ? full mode only ?). */
	2067	ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType);
	2068
	2069	/*! ZSTD_DCtx_refPrefix_advanced() :
	2070	* Same as ZSTD_DCtx_refPrefix(), but gives finer control over
	2071	* how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */
	2072	ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType);
	2073
	2074	/*! ZSTD_DCtx_setMaxWindowSize() :
	2075	* Refuses allocating internal buffers for frames requiring a window size larger than provided limit.
	2076	* This protects a decoder context from reserving too much memory for itself (potential attack scenario).
	2077	* This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
	2078	* By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT)
	2079	* @return : 0, or an error code (which can be tested using ZSTD_isError()).
	2080	*/
	2081	ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize);
	2082
	2083	/*! ZSTD_DCtx_getParameter() :
	2084	* Get the requested decompression parameter value, selected by enum ZSTD_dParameter,
	2085	* and store it into int* value.
	2086	* @return : 0, or an error code (which can be tested with ZSTD_isError()).
	2087	*/
	2088	ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value);
	2089
	2090	/* ZSTD_d_format
	2091	* experimental parameter,
	2092	* allowing selection between ZSTD_format_e input compression formats
	2093	*/
	2094	#define ZSTD_d_format ZSTD_d_experimentalParam1
	2095	/* ZSTD_d_stableOutBuffer
	2096	* Experimental parameter.
	2097	* Default is 0 == disabled. Set to 1 to enable.
	2098	*
	2099	* Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same
	2100	* between calls, except for the modifications that zstd makes to pos (the
	2101	* caller must not modify pos). This is checked by the decompressor, and
	2102	* decompression will fail if it ever changes. Therefore the ZSTD_outBuffer
	2103	* MUST be large enough to fit the entire decompressed frame. This will be
	2104	* checked when the frame content size is known. The data in the ZSTD_outBuffer
	2105	* in the range [dst, dst + pos) MUST not be modified during decompression
	2106	* or you will get data corruption.
	2107	*
	2108	* When this flags is enabled zstd won't allocate an output buffer, because
	2109	* it can write directly to the ZSTD_outBuffer, but it will still allocate
	2110	* an input buffer large enough to fit any compressed block. This will also
	2111	* avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer.
	2112	* If you need to avoid the input buffer allocation use the buffer-less
	2113	* streaming API.
	2114	*
	2115	* NOTE: So long as the ZSTD_outBuffer always points to valid memory, using
	2116	* this flag is ALWAYS memory safe, and will never access out-of-bounds
	2117	* memory. However, decompression WILL fail if you violate the preconditions.
	2118	*
	2119	* WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST
	2120	* not be modified during decompression or you will get data corruption. This
	2121	* is because zstd needs to reference data in the ZSTD_outBuffer to regenerate
	2122	* matches. Normally zstd maintains its own buffer for this purpose, but passing
	2123	* this flag tells zstd to use the user provided buffer.
	2124	*/
	2125	#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2
	2126
	2127	/* ZSTD_d_forceIgnoreChecksum
	2128	* Experimental parameter.
	2129	* Default is 0 == disabled. Set to 1 to enable
	2130	*
	2131	* Tells the decompressor to skip checksum validation during decompression, regardless
	2132	* of whether checksumming was specified during compression. This offers some
	2133	* slight performance benefits, and may be useful for debugging.
	2134	* Param has values of type ZSTD_forceIgnoreChecksum_e
	2135	*/
	2136	#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3
	2137
	2138	/* ZSTD_d_refMultipleDDicts
	2139	* Experimental parameter.
	2140	* Default is 0 == disabled. Set to 1 to enable
	2141	*
	2142	* If enabled and dctx is allocated on the heap, then additional memory will be allocated
	2143	* to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict()
	2144	* using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead
	2145	* store all references. At decompression time, the appropriate dictID is selected
	2146	* from the set of DDicts based on the dictID in the frame.
	2147	*
	2148	* Usage is simply calling ZSTD_refDDict() on multiple dict buffers.
	2149	*
	2150	* Param has values of byte ZSTD_refMultipleDDicts_e
	2151	*
	2152	* WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory
	2153	* allocation for the hash table. ZSTD_freeDCtx() also frees this memory.
	2154	* Memory is allocated as per ZSTD_DCtx::customMem.
	2155	*
	2156	* Although this function allocates memory for the table, the user is still responsible for
	2157	* memory management of the underlying ZSTD_DDict* themselves.
	2158	*/
	2159	#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4
	2160
	2161
	2162	/*! ZSTD_DCtx_setFormat() :
	2163	* This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter().
	2164	* Instruct the decoder context about what kind of data to decode next.
	2165	* This instruction is mandatory to decode data without a fully-formed header,
	2166	* such ZSTD_f_zstd1_magicless for example.
	2167	* @return : 0, or an error code (which can be tested using ZSTD_isError()). */
	2168	ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead")
	2169	size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
	2170
	2171	/*! ZSTD_decompressStream_simpleArgs() :
	2172	* Same as ZSTD_decompressStream(),
	2173	* but using only integral types as arguments.
	2174	* This can be helpful for binders from dynamic languages
	2175	* which have troubles handling structures containing memory pointers.
	2176	*/
	2177	ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs (
	2178	ZSTD_DCtx* dctx,
	2179	void* dst, size_t dstCapacity, size_t* dstPos,
	2180	const void* src, size_t srcSize, size_t* srcPos);
	2181
	2182
	2183	/********************************************************************
	2184	* Advanced streaming functions
	2185	* Warning : most of these functions are now redundant with the Advanced API.
	2186	* Once Advanced API reaches "stable" status,
	2187	* redundant functions will be deprecated, and then at some point removed.
	2188	********************************************************************/
	2189
	2190	/===== Advanced Streaming compression functions =====/
	2191
	2192	/*! ZSTD_initCStream_srcSize() :
	2193	* This function is DEPRECATED, and equivalent to:
	2194	* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
	2195	* ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
	2196	* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
	2197	* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
	2198	*
	2199	* pledgedSrcSize must be correct. If it is not known at init time, use
	2200	* ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs,
	2201	* "0" also disables frame content size field. It may be enabled in the future.
	2202	* This prototype will generate compilation warnings.
	2203	*/
	2204	ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
	2205	size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
	2206	int compressionLevel,
	2207	unsigned long long pledgedSrcSize);
	2208
	2209	/*! ZSTD_initCStream_usingDict() :
	2210	* This function is DEPRECATED, and is equivalent to:
	2211	* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
	2212	* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
	2213	* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
	2214	*
	2215	* Creates of an internal CDict (incompatible with static CCtx), except if
	2216	* dict == NULL or dictSize < 8, in which case no dict is used.
	2217	* Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
	2218	* it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
	2219	* This prototype will generate compilation warnings.
	2220	*/
	2221	ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
	2222	size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
	2223	const void* dict, size_t dictSize,
	2224	int compressionLevel);
	2225
	2226	/*! ZSTD_initCStream_advanced() :
	2227	* This function is DEPRECATED, and is approximately equivalent to:
	2228	* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
	2229	* // Pseudocode: Set each zstd parameter and leave the rest as-is.
	2230	* for ((param, value) : params) {
	2231	* ZSTD_CCtx_setParameter(zcs, param, value);
	2232	* }
	2233	* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
	2234	* ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
	2235	*
	2236	* dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
	2237	* pledgedSrcSize must be correct.
	2238	* If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
	2239	* This prototype will generate compilation warnings.
	2240	*/
	2241	ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
	2242	size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
	2243	const void* dict, size_t dictSize,
	2244	ZSTD_parameters params,
	2245	unsigned long long pledgedSrcSize);
	2246
	2247	/*! ZSTD_initCStream_usingCDict() :
	2248	* This function is DEPRECATED, and equivalent to:
	2249	* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
	2250	* ZSTD_CCtx_refCDict(zcs, cdict);
	2251	*
	2252	* note : cdict will just be referenced, and must outlive compression session
	2253	* This prototype will generate compilation warnings.
	2254	*/
	2255	ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
	2256	size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
	2257
	2258	/*! ZSTD_initCStream_usingCDict_advanced() :
	2259	* This function is DEPRECATED, and is approximately equivalent to:
	2260	* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
	2261	* // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
	2262	* for ((fParam, value) : fParams) {
	2263	* ZSTD_CCtx_setParameter(zcs, fParam, value);
	2264	* }
	2265	* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
	2266	* ZSTD_CCtx_refCDict(zcs, cdict);
	2267	*
	2268	* same as ZSTD_initCStream_usingCDict(), with control over frame parameters.
	2269	* pledgedSrcSize must be correct. If srcSize is not known at init time, use
	2270	* value ZSTD_CONTENTSIZE_UNKNOWN.
	2271	* This prototype will generate compilation warnings.
	2272	*/
	2273	ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
	2274	size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
	2275	const ZSTD_CDict* cdict,
	2276	ZSTD_frameParameters fParams,
	2277	unsigned long long pledgedSrcSize);
	2278
	2279	/*! ZSTD_resetCStream() :
	2280	* This function is DEPRECATED, and is equivalent to:
	2281	* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
	2282	* ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
	2283	* Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but
	2284	* ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be
	2285	* explicitly specified.
	2286	*
	2287	* start a new frame, using same parameters from previous frame.
	2288	* This is typically useful to skip dictionary loading stage, since it will re-use it in-place.
	2289	* Note that zcs must be init at least once before using ZSTD_resetCStream().
	2290	* If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
	2291	* If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
	2292	* For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
	2293	* but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
	2294	* @return : 0, or an error code (which can be tested using ZSTD_isError())
	2295	* This prototype will generate compilation warnings.
	2296	*/
	2297	ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
	2298	size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
	2299
	2300
	2301	typedef struct {
	2302	unsigned long long ingested; /* nb input bytes read and buffered */
	2303	unsigned long long consumed; /* nb input bytes actually compressed */
	2304	unsigned long long produced; /* nb of compressed bytes generated and buffered */
	2305	unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */
	2306	unsigned currentJobID; /* MT only : latest started job nb */
	2307	unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */
	2308	} ZSTD_frameProgression;
	2309
	2310	/* ZSTD_getFrameProgression() :
	2311	* tells how much data has been ingested (read from input)
	2312	* consumed (input actually compressed) and produced (output) for current frame.
	2313	* Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed.
	2314	* Aggregates progression inside active worker threads.
	2315	*/
	2316	ZSTDLIB_STATIC_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx);
	2317
	2318	/*! ZSTD_toFlushNow() :
	2319	* Tell how many bytes are ready to be flushed immediately.
	2320	* Useful for multithreading scenarios (nbWorkers >= 1).
	2321	* Probe the oldest active job, defined as oldest job not yet entirely flushed,
	2322	* and check its output buffer.
	2323	* @return : amount of data stored in oldest job and ready to be flushed immediately.
	2324	* if @return == 0, it means either :
	2325	* + there is no active job (could be checked with ZSTD_frameProgression()), or
	2326	* + oldest job is still actively compressing data,
	2327	* but everything it has produced has also been flushed so far,
	2328	* therefore flush speed is limited by production speed of oldest job
	2329	* irrespective of the speed of concurrent (and newer) jobs.
	2330	*/
	2331	ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
	2332
	2333
	2334	/===== Advanced Streaming decompression functions =====/
	2335
	2336	/*!
	2337	* This function is deprecated, and is equivalent to:
	2338	*
	2339	* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
	2340	* ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
	2341	*
	2342	* note: no dictionary will be used if dict == NULL or dictSize < 8
	2343	* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
	2344	*/
	2345	ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
	2346
	2347	/*!
	2348	* This function is deprecated, and is equivalent to:
	2349	*
	2350	* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
	2351	* ZSTD_DCtx_refDDict(zds, ddict);
	2352	*
	2353	* note : ddict is referenced, it must outlive decompression session
	2354	* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
	2355	*/
	2356	ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
	2357
	2358	/*!
	2359	* This function is deprecated, and is equivalent to:
	2360	*
	2361	* ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
	2362	*
	2363	* re-use decompression parameters from previous init; saves dictionary loading
	2364	* Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
	2365	*/
	2366	ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
	2367
	2368
	2369	/*********************************************************************
	2370	* Buffer-less and synchronous inner streaming functions
	2371	*
	2372	* This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
	2373	* But it's also a complex one, with several restrictions, documented below.
	2374	* Prefer normal streaming API for an easier experience.
	2375	********************************************************************* */
	2376
	2377	/**
	2378	Buffer-less streaming compression (synchronous mode)
	2379
	2380	A ZSTD_CCtx object is required to track streaming operations.
	2381	Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
	2382	ZSTD_CCtx object can be re-used multiple times within successive compression operations.
	2383
	2384	Start by initializing a context.
	2385	Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression.
	2386	It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()
	2387
	2388	Then, consume your input using ZSTD_compressContinue().
	2389	There are some important considerations to keep in mind when using this advanced function :
	2390	- ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only.
	2391	- Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks.
	2392	- Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario.
	2393	Worst case evaluation is provided by ZSTD_compressBound().
	2394	ZSTD_compressContinue() doesn't guarantee recover after a failed compression.
	2395	- ZSTD_compressContinue() presumes prior input *is still accessible and unmodified* (up to maximum distance size, see WindowLog).
	2396	It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks)
	2397	- ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps.
	2398	In which case, it will "discard" the relevant memory section from its history.
	2399
	2400	Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
	2401	It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
	2402	Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders.
	2403
	2404	`ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again.
	2405	*/
	2406
	2407	/===== Buffer-less streaming compression functions =====/
	2408	ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
	2409	ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
	2410	ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /*< note: fails if cdict==NULL /
	2411	ZSTDLIB_STATIC_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /*< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN /
	2412
	2413	ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
	2414	ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
	2415
	2416	/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */
	2417	ZSTD_DEPRECATED("use advanced API to access custom parameters")
	2418	size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /*< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN /
	2419	ZSTD_DEPRECATED("use advanced API to access custom parameters")
	2420	size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
	2421	/**
	2422	Buffer-less streaming decompression (synchronous mode)
	2423
	2424	A ZSTD_DCtx object is required to track streaming operations.
	2425	Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
	2426	A ZSTD_DCtx object can be re-used multiple times.
	2427
	2428	First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader().
	2429	Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
	2430	Data fragment must be large enough to ensure successful decoding.
	2431	`ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
	2432	@result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
	2433	>0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
	2434	errorCode, which can be tested using ZSTD_isError().
	2435
	2436	It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
	2437	such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`).
	2438	Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information.
	2439	As a consequence, check that values remain within valid application range.
	2440	For example, do not allocate memory blindly, check that `windowSize` is within expectation.
	2441	Each application can set its own limits, depending on local restrictions.
	2442	For extended interoperability, it is recommended to support `windowSize` of at least 8 MB.
	2443
	2444	ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes.
	2445	ZSTD_decompressContinue() is very sensitive to contiguity,
	2446	if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place,
	2447	or that previous contiguous segment is large enough to properly handle maximum back-reference distance.
	2448	There are multiple ways to guarantee this condition.
	2449
	2450	The most memory efficient way is to use a round buffer of sufficient size.
	2451	Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(),
	2452	which can @return an error code if required value is too large for current system (in 32-bits mode).
	2453	In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one,
	2454	up to the moment there is not enough room left in the buffer to guarantee decoding another full block,
	2455	which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`.
	2456	At which point, decoding can resume from the beginning of the buffer.
	2457	Note that already decoded data stored in the buffer should be flushed before being overwritten.
	2458
	2459	There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory.
	2460
	2461	Finally, if you control the compression process, you can also ignore all buffer size rules,
	2462	as long as the encoder and decoder progress in "lock-step",
	2463	aka use exactly the same buffer sizes, break contiguity at the same place, etc.
	2464
	2465	Once buffers are setup, start decompression, with ZSTD_decompressBegin().
	2466	If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict().
	2467
	2468	Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
	2469	ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
	2470	ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
	2471
	2472	@result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
	2473	It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item.
	2474	It can also be an error code, which can be tested with ZSTD_isError().
	2475
	2476	A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
	2477	Context can then be reset to start a new decompression.
	2478
	2479	Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
	2480	This information is not required to properly decode a frame.
	2481
	2482	== Special case : skippable frames ==
	2483
	2484	Skippable frames allow integration of user-defined data into a flow of concatenated frames.
	2485	Skippable frames will be ignored (skipped) by decompressor.
	2486	The format of skippable frames is as follows :
	2487	a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F
	2488	b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits
	2489	c) Frame Content - any content (User Data) of length equal to Frame Size
	2490	For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame.
	2491	For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content.
	2492	*/
	2493
	2494	/===== Buffer-less streaming decompression functions =====/
	2495	typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
	2496	typedef struct {
	2497	unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */
	2498	unsigned long long windowSize; /* can be very large, up to <= frameContentSize */
	2499	unsigned blockSizeMax;
	2500	ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
	2501	unsigned headerSize;
	2502	unsigned dictID;
	2503	unsigned checksumFlag;
	2504	} ZSTD_frameHeader;
	2505
	2506	/*! ZSTD_getFrameHeader() :
	2507	* decode Frame Header, or requires larger `srcSize`.
	2508	* @return : 0, `zfhPtr` is correctly filled,
	2509	* >0, `srcSize` is too small, value is wanted `srcSize` amount,
	2510	* or an error code, which can be tested using ZSTD_isError() */
	2511	ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /*< doesn't consume input /
	2512	/*! ZSTD_getFrameHeader_advanced() :
	2513	* same as ZSTD_getFrameHeader(),
	2514	* with added capability to select a format (like ZSTD_f_zstd1_magicless) */
	2515	ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
	2516	ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /*< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN /
	2517
	2518	ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
	2519	ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
	2520	ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
	2521
	2522	ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
	2523	ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
	2524
	2525	/* misc */
	2526	ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
	2527	typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
	2528	ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
	2529
	2530
	2531
	2532
	2533	/* ============================ */
	2534	/** Block level API */
	2535	/* ============================ */
	2536
	2537	/*!
	2538	Block functions produce and decode raw zstd blocks, without frame metadata.
	2539	Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
	2540	But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
	2541
	2542	A few rules to respect :
	2543	- Compressing and decompressing require a context structure
	2544	+ Use ZSTD_createCCtx() and ZSTD_createDCtx()
	2545	- It is necessary to init context before starting
	2546	+ compression : any ZSTD_compressBegin*() variant, including with dictionary
	2547	+ decompression : any ZSTD_decompressBegin*() variant, including with dictionary
	2548	+ copyCCtx() and copyDCtx() can be used too
	2549	- Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
	2550	+ If input is larger than a block size, it's necessary to split input data into multiple blocks
	2551	+ For inputs larger than a single block, consider using regular ZSTD_compress() instead.
	2552	Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block.
	2553	- When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) !
	2554	===> In which case, nothing is produced into `dst` !
	2555	+ User __must__ test for such outcome and deal directly with uncompressed data
	2556	+ A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0.
	2557	Doing so would mess up with statistics history, leading to potential data corruption.
	2558	+ ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !!
	2559	+ In case of multiple successive blocks, should some of them be uncompressed,
	2560	decoder must be informed of their existence in order to follow proper history.
	2561	Use ZSTD_insertBlock() for such a case.
	2562	*/
	2563
	2564	/===== Raw zstd block functions =====/
	2565	ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx);
	2566	ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
	2567	ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
	2568	ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /*< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. /
	2569
	2570
	2571	#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
	2572
	2573	#if defined (__cplusplus)
	2574	}
	2575	#endif

stage1/zstd/lib/zstd_errors.h created+95

...	...	@@ -0,0 +1,95 @@
	1	/*
	2	* Copyright (c) Yann Collet, Facebook, Inc.
	3	* All rights reserved.
	4	*
	5	* This source code is licensed under both the BSD-style license (found in the
	6	* LICENSE file in the root directory of this source tree) and the GPLv2 (found
	7	* in the COPYING file in the root directory of this source tree).
	8	* You may select, at your option, one of the above-listed licenses.
	9	*/
	10
	11	#ifndef ZSTD_ERRORS_H_398273423
	12	#define ZSTD_ERRORS_H_398273423
	13
	14	#if defined (__cplusplus)
	15	extern "C" {
	16	#endif
	17
	18	/===== dependency =====/
	19	#include <stddef.h> /* size_t */
	20
	21
	22	/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */
	23	#ifndef ZSTDERRORLIB_VISIBILITY
	24	# if defined(__GNUC__) && (__GNUC__ >= 4)
	25	# define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default")))
	26	# else
	27	# define ZSTDERRORLIB_VISIBILITY
	28	# endif
	29	#endif
	30	#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
	31	# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY
	32	#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
	33	# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
	34	#else
	35	# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY
	36	#endif
	37
	38	/-********************************************
	39	* Error codes list
	40	-********************************************
	41	* Error codes _values_ are pinned down since v1.3.1 only.
	42	* Therefore, don't rely on values if you may link to any version < v1.3.1.
	43	*
	44	* Only values < 100 are considered stable.
	45	*
	46	* note 1 : this API shall be used with static linking only.
	47	* dynamic linking is not yet officially supported.
	48	* note 2 : Prefer relying on the enum than on its value whenever possible
	49	* This is the only supported way to use the error list < v1.3.1
	50	* note 3 : ZSTD_isError() is always correct, whatever the library version.
	51	**********************************************/
	52	typedef enum {
	53	ZSTD_error_no_error = 0,
	54	ZSTD_error_GENERIC = 1,
	55	ZSTD_error_prefix_unknown = 10,
	56	ZSTD_error_version_unsupported = 12,
	57	ZSTD_error_frameParameter_unsupported = 14,
	58	ZSTD_error_frameParameter_windowTooLarge = 16,
	59	ZSTD_error_corruption_detected = 20,
	60	ZSTD_error_checksum_wrong = 22,
	61	ZSTD_error_dictionary_corrupted = 30,
	62	ZSTD_error_dictionary_wrong = 32,
	63	ZSTD_error_dictionaryCreation_failed = 34,
	64	ZSTD_error_parameter_unsupported = 40,
	65	ZSTD_error_parameter_outOfBound = 42,
	66	ZSTD_error_tableLog_tooLarge = 44,
	67	ZSTD_error_maxSymbolValue_tooLarge = 46,
	68	ZSTD_error_maxSymbolValue_tooSmall = 48,
	69	ZSTD_error_stage_wrong = 60,
	70	ZSTD_error_init_missing = 62,
	71	ZSTD_error_memory_allocation = 64,
	72	ZSTD_error_workSpace_tooSmall= 66,
	73	ZSTD_error_dstSize_tooSmall = 70,
	74	ZSTD_error_srcSize_wrong = 72,
	75	ZSTD_error_dstBuffer_null = 74,
	76	/* following error codes are __NOT STABLE__, they can be removed or changed in future versions */
	77	ZSTD_error_frameIndex_tooLarge = 100,
	78	ZSTD_error_seekableIO = 102,
	79	ZSTD_error_dstBuffer_wrong = 104,
	80	ZSTD_error_srcBuffer_wrong = 105,
	81	ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
	82	} ZSTD_ErrorCode;
	83
	84	/*! ZSTD_getErrorCode() :
	85	convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
	86	which can be used to compare with enum list published above */
	87	ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
	88	ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /*< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument /
	89
	90
	91	#if defined (__cplusplus)
	92	}
	93	#endif
	94
	95	#endif /* ZSTD_ERRORS_H_398273423 */