1 /*
2 frameTest - test tool for lz4frame
3 Copyright (C) Yann Collet 2014-2020
4
5 GPL v2 License
6
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
11
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
16
17 You should have received a copy of the GNU General Public License along
18 with this program; if not, write to the Free Software Foundation, Inc.,
19 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21 You can contact the author at :
22 - LZ4 homepage : http://www.lz4.org
23 - LZ4 source repository : https://github.com/lz4/lz4
24 */
25
26 /*-************************************
27 * Compiler specific
28 **************************************/
29 #ifdef _MSC_VER /* Visual Studio */
30 # pragma warning(disable : 26451) /* disable: Arithmetic overflow */
31 #endif
32
33
34 /*-************************************
35 * Includes
36 **************************************/
37 #include "util.h" /* U32 */
38 #include <stdlib.h> /* malloc, free */
39 #include <stdio.h> /* fprintf */
40 #include <string.h> /* strcmp */
41 #include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */
42 #include <assert.h>
43 #include "lz4frame.h" /* included multiple times to test correctness/safety */
44 #include "lz4frame.h"
45 #define LZ4F_STATIC_LINKING_ONLY
46 #include "lz4frame.h"
47 #include "lz4frame.h"
48 #define LZ4_STATIC_LINKING_ONLY /* LZ4_DISTANCE_MAX */
49 #include "lz4.h" /* LZ4_VERSION_STRING */
50 #define XXH_STATIC_LINKING_ONLY
51 #include "xxhash.h" /* XXH64 */
52
53
54 /* unoptimized version; solves endianess & alignment issues */
FUZ_writeLE32(void * dstVoidPtr,U32 value32)55 static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32)
56 {
57 BYTE* dstPtr = (BYTE*)dstVoidPtr;
58 dstPtr[0] = (BYTE) value32;
59 dstPtr[1] = (BYTE)(value32 >> 8);
60 dstPtr[2] = (BYTE)(value32 >> 16);
61 dstPtr[3] = (BYTE)(value32 >> 24);
62 }
63
64
65 /*-************************************
66 * Constants
67 **************************************/
68 #define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
69
70 #define KB *(1U<<10)
71 #define MB *(1U<<20)
72 #define GB *(1U<<30)
73
74 static const U32 nbTestsDefault = 256 KB;
75 #define FUZ_COMPRESSIBILITY_DEFAULT 50
76 static const U32 prime1 = 2654435761U;
77 static const U32 prime2 = 2246822519U;
78
79
80 /*-************************************
81 * Macros
82 **************************************/
83 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
84 #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
85 #define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \
86 if ((FUZ_GetClockSpan(g_clockTime) > refreshRate) || (displayLevel>=4)) \
87 { g_clockTime = clock(); DISPLAY(__VA_ARGS__); \
88 if (displayLevel>=4) fflush(stdout); } }
89 static const clock_t refreshRate = CLOCKS_PER_SEC / 6;
90 static clock_t g_clockTime = 0;
91
92
93 /*-***************************************
94 * Local Parameters
95 *****************************************/
96 static U32 no_prompt = 0;
97 static U32 displayLevel = 2;
98 static U32 use_pause = 0;
99
100
101 /*-*******************************************************
102 * Fuzzer functions
103 *********************************************************/
104 #define MIN(a,b) ( (a) < (b) ? (a) : (b) )
105 #define MAX(a,b) ( (a) > (b) ? (a) : (b) )
106
FUZ_GetClockSpan(clock_t clockStart)107 static clock_t FUZ_GetClockSpan(clock_t clockStart)
108 {
109 return clock() - clockStart; /* works even if overflow; max span ~ 30 mn */
110 }
111
112
113 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
FUZ_rand(unsigned int * src)114 unsigned int FUZ_rand(unsigned int* src)
115 {
116 U32 rand32 = *src;
117 rand32 *= prime1;
118 rand32 += prime2;
119 rand32 = FUZ_rotl32(rand32, 13);
120 *src = rand32;
121 return rand32 >> 5;
122 }
123
124
125 #define FUZ_RAND15BITS (FUZ_rand(seed) & 0x7FFF)
126 #define FUZ_RANDLENGTH ( (FUZ_rand(seed) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15)
FUZ_fillCompressibleNoiseBuffer(void * buffer,size_t bufferSize,double proba,U32 * seed)127 static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, double proba, U32* seed)
128 {
129 BYTE* BBuffer = (BYTE*)buffer;
130 size_t pos = 0;
131 U32 P32 = (U32)(32768 * proba);
132
133 /* First Byte */
134 BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
135
136 while (pos < bufferSize) {
137 /* Select : Literal (noise) or copy (within 64K) */
138 if (FUZ_RAND15BITS < P32) {
139 /* Copy (within 64K) */
140 size_t const lengthRand = FUZ_RANDLENGTH + 4;
141 size_t const length = MIN(lengthRand, bufferSize - pos);
142 size_t const end = pos + length;
143 size_t const offsetRand = FUZ_RAND15BITS + 1;
144 size_t const offset = MIN(offsetRand, pos);
145 size_t match = pos - offset;
146 while (pos < end) BBuffer[pos++] = BBuffer[match++];
147 } else {
148 /* Literal (noise) */
149 size_t const lengthRand = FUZ_RANDLENGTH + 4;
150 size_t const length = MIN(lengthRand, bufferSize - pos);
151 size_t const end = pos + length;
152 while (pos < end) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5);
153 } }
154 }
155
156
FUZ_highbit(U32 v32)157 static unsigned FUZ_highbit(U32 v32)
158 {
159 unsigned nbBits = 0;
160 if (v32==0) return 0;
161 while (v32) {v32 >>= 1; nbBits ++;}
162 return nbBits;
163 }
164
165
166 /*-*******************************************************
167 * Tests
168 *********************************************************/
169 #define CHECK_V(v,f) v = f; if (LZ4F_isError(v)) { fprintf(stderr, "%s \n", LZ4F_getErrorName(v)); goto _output_error; }
170 #define CHECK(f) { LZ4F_errorCode_t const CHECK_V(err_ , f); }
171
basicTests(U32 seed,double compressibility)172 int basicTests(U32 seed, double compressibility)
173 {
174 #define COMPRESSIBLE_NOISE_LENGTH (2 MB)
175 void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
176 size_t const cBuffSize = LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL);
177 void* const compressedBuffer = malloc(cBuffSize);
178 void* const decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
179 U32 randState = seed;
180 size_t cSize, testSize;
181 LZ4F_decompressionContext_t dCtx = NULL;
182 LZ4F_compressionContext_t cctx = NULL;
183 U64 crcOrig;
184 int basicTests_error = 0;
185 LZ4F_preferences_t prefs;
186 memset(&prefs, 0, sizeof(prefs));
187
188 if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
189 DISPLAY("allocation error, not enough memory to start fuzzer tests \n");
190 goto _output_error;
191 }
192 FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
193 crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
194
195 /* LZ4F_compressBound() : special case : srcSize == 0 */
196 DISPLAYLEVEL(3, "LZ4F_compressBound(0) = ");
197 { size_t const cBound = LZ4F_compressBound(0, NULL);
198 if (cBound < 64 KB) goto _output_error;
199 DISPLAYLEVEL(3, " %u \n", (U32)cBound);
200 }
201
202 /* LZ4F_compressBound() : special case : automatic flushing enabled */
203 DISPLAYLEVEL(3, "LZ4F_compressBound(1 KB, autoFlush=1) = ");
204 { size_t cBound;
205 LZ4F_preferences_t autoFlushPrefs;
206 memset(&autoFlushPrefs, 0, sizeof(autoFlushPrefs));
207 autoFlushPrefs.autoFlush = 1;
208 cBound = LZ4F_compressBound(1 KB, &autoFlushPrefs);
209 if (cBound > 64 KB) goto _output_error;
210 DISPLAYLEVEL(3, " %u \n", (U32)cBound);
211 }
212
213 /* LZ4F_compressBound() : special case : automatic flushing disabled */
214 DISPLAYLEVEL(3, "LZ4F_compressBound(1 KB, autoFlush=0) = ");
215 { size_t const cBound = LZ4F_compressBound(1 KB, &prefs);
216 if (cBound < 64 KB) goto _output_error;
217 DISPLAYLEVEL(3, " %u \n", (U32)cBound);
218 }
219
220 /* Special case : null-content frame */
221 testSize = 0;
222 DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : ");
223 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL));
224 DISPLAYLEVEL(3, "null content encoded into a %u bytes frame \n", (unsigned)cSize);
225
226 DISPLAYLEVEL(3, "LZ4F_createDecompressionContext \n");
227 CHECK ( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
228
229 DISPLAYLEVEL(3, "LZ4F_getFrameInfo on null-content frame (#157) \n");
230 assert(cSize >= LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH);
231 { LZ4F_frameInfo_t frame_info;
232 size_t const fhs = LZ4F_headerSize(compressedBuffer, LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH);
233 size_t avail_in = fhs;
234 CHECK( fhs );
235 CHECK( LZ4F_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in) );
236 if (avail_in != fhs) goto _output_error; /* must consume all, since header size is supposed to be exact */
237 }
238
239 DISPLAYLEVEL(3, "LZ4F_freeDecompressionContext \n");
240 CHECK( LZ4F_freeDecompressionContext(dCtx) );
241 dCtx = NULL;
242
243 /* test one-pass frame compression */
244 testSize = COMPRESSIBLE_NOISE_LENGTH;
245
246 DISPLAYLEVEL(3, "LZ4F_compressFrame, using fast level -3 : ");
247 { LZ4F_preferences_t fastCompressPrefs;
248 memset(&fastCompressPrefs, 0, sizeof(fastCompressPrefs));
249 fastCompressPrefs.compressionLevel = -3;
250 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, &fastCompressPrefs));
251 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
252 }
253
254 DISPLAYLEVEL(3, "LZ4F_compressFrame, using default preferences : ");
255 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL));
256 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
257
258 DISPLAYLEVEL(3, "Decompression test : \n");
259 { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
260 size_t compressedBufferSize = cSize;
261
262 CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
263
264 DISPLAYLEVEL(3, "Single Pass decompression : ");
265 CHECK( LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL) );
266 { U64 const crcDest = XXH64(decodedBuffer, decodedBufferSize, 1);
267 if (crcDest != crcOrig) goto _output_error; }
268 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedBufferSize);
269
270 DISPLAYLEVEL(3, "Reusing decompression context \n");
271 { size_t const missingBytes = 4;
272 size_t iSize = compressedBufferSize - missingBytes;
273 const BYTE* cBuff = (const BYTE*) compressedBuffer;
274 BYTE* const ostart = (BYTE*)decodedBuffer;
275 BYTE* op = ostart;
276 BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
277 size_t decResult, oSize = COMPRESSIBLE_NOISE_LENGTH;
278 DISPLAYLEVEL(3, "Missing last %u bytes : ", (U32)missingBytes);
279 CHECK_V(decResult, LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL));
280 if (decResult != missingBytes) {
281 DISPLAY("%u bytes missing != %u bytes requested \n", (U32)missingBytes, (U32)decResult);
282 goto _output_error;
283 }
284 DISPLAYLEVEL(3, "indeed, requests %u bytes \n", (unsigned)decResult);
285 cBuff += iSize;
286 iSize = decResult;
287 op += oSize;
288 oSize = (size_t)(oend-op);
289 decResult = LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL);
290 if (decResult != 0) goto _output_error; /* should finish now */
291 op += oSize;
292 if (op>oend) { DISPLAY("decompression write overflow \n"); goto _output_error; }
293 { U64 const crcDest = XXH64(decodedBuffer, (size_t)(op-ostart), 1);
294 if (crcDest != crcOrig) goto _output_error;
295 } }
296
297 { size_t oSize = 0;
298 size_t iSize = 0;
299 LZ4F_frameInfo_t fi;
300 const BYTE* ip = (BYTE*)compressedBuffer;
301
302 DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : ");
303 CHECK( LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL) );
304 //DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode);
305 DISPLAYLEVEL(3, " OK \n");
306
307 DISPLAYLEVEL(3, "LZ4F_getFrameInfo on zero-size input : ");
308 { size_t nullSize = 0;
309 size_t const fiError = LZ4F_getFrameInfo(dCtx, &fi, ip, &nullSize);
310 if (LZ4F_getErrorCode(fiError) != LZ4F_ERROR_frameHeader_incomplete) {
311 DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n",
312 LZ4F_getErrorName(fiError));
313 goto _output_error;
314 }
315 DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(fiError));
316 }
317
318 DISPLAYLEVEL(3, "LZ4F_getFrameInfo on not enough input : ");
319 { size_t inputSize = 6;
320 size_t const fiError = LZ4F_getFrameInfo(dCtx, &fi, ip, &inputSize);
321 if (LZ4F_getErrorCode(fiError) != LZ4F_ERROR_frameHeader_incomplete) {
322 DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n", LZ4F_getErrorName(fiError));
323 goto _output_error;
324 }
325 DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(fiError));
326 }
327
328 DISPLAYLEVEL(3, "LZ4F_getFrameInfo on enough input : ");
329 iSize = LZ4F_headerSize(ip, LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH);
330 CHECK( iSize );
331 CHECK( LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize) );
332 DISPLAYLEVEL(3, " correctly decoded \n");
333 }
334
335 DISPLAYLEVEL(3, "Decode a buggy input : ");
336 assert(COMPRESSIBLE_NOISE_LENGTH > 64);
337 assert(cSize > 48);
338 memcpy(decodedBuffer, (char*)compressedBuffer+16, 32); /* save correct data */
339 memcpy((char*)compressedBuffer+16, (const char*)decodedBuffer+32, 32); /* insert noise */
340 { size_t dbSize = COMPRESSIBLE_NOISE_LENGTH;
341 size_t cbSize = cSize;
342 size_t const decompressError = LZ4F_decompress(dCtx, decodedBuffer, &dbSize,
343 compressedBuffer, &cbSize,
344 NULL);
345 if (!LZ4F_isError(decompressError)) goto _output_error;
346 DISPLAYLEVEL(3, "error detected : %s \n", LZ4F_getErrorName(decompressError));
347 }
348 memcpy((char*)compressedBuffer+16, decodedBuffer, 32); /* restore correct data */
349
350 DISPLAYLEVEL(3, "Reset decompression context, since it's left in error state \n");
351 LZ4F_resetDecompressionContext(dCtx); /* always successful */
352
353 DISPLAYLEVEL(3, "Byte after byte : ");
354 { BYTE* const ostart = (BYTE*)decodedBuffer;
355 BYTE* op = ostart;
356 BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
357 const BYTE* ip = (const BYTE*) compressedBuffer;
358 const BYTE* const iend = ip + cSize;
359 while (ip < iend) {
360 size_t oSize = (size_t)(oend-op);
361 size_t iSize = 1;
362 CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
363 op += oSize;
364 ip += iSize;
365 }
366 { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
367 if (crcDest != crcOrig) goto _output_error;
368 }
369 DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), (unsigned)COMPRESSIBLE_NOISE_LENGTH);
370 }
371 }
372
373 DISPLAYLEVEL(3, "Using 64 KB block : ");
374 prefs.frameInfo.blockSizeID = LZ4F_max64KB;
375 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
376 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs));
377 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
378
379 DISPLAYLEVEL(3, "without checksum : ");
380 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
381 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs));
382 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
383
384 DISPLAYLEVEL(3, "Using 256 KB block : ");
385 prefs.frameInfo.blockSizeID = LZ4F_max256KB;
386 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
387 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs));
388 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
389
390 DISPLAYLEVEL(3, "Decompression test : \n");
391 { size_t const decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
392 unsigned const maxBits = FUZ_highbit((U32)decodedBufferSize);
393 BYTE* const ostart = (BYTE*)decodedBuffer;
394 BYTE* op = ostart;
395 BYTE* const oend = ostart + COMPRESSIBLE_NOISE_LENGTH;
396 const BYTE* ip = (const BYTE*)compressedBuffer;
397 const BYTE* const iend = (const BYTE*)compressedBuffer + cSize;
398
399 DISPLAYLEVEL(3, "random segment sizes : ");
400 while (ip < iend) {
401 unsigned const nbBits = FUZ_rand(&randState) % maxBits;
402 size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
403 size_t oSize = (size_t)(oend-op);
404 if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip);
405 CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
406 op += oSize;
407 ip += iSize;
408 }
409 { size_t const decodedSize = (size_t)(op - ostart);
410 U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1);
411 if (crcDest != crcOrig) goto _output_error;
412 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
413 }
414
415 CHECK( LZ4F_freeDecompressionContext(dCtx) );
416 dCtx = NULL;
417 }
418
419 DISPLAYLEVEL(3, "without checksum : ");
420 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
421 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
422 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
423
424 DISPLAYLEVEL(3, "Using 1 MB block : ");
425 prefs.frameInfo.blockSizeID = LZ4F_max1MB;
426 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
427 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
428 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
429
430 DISPLAYLEVEL(3, "without frame checksum : ");
431 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
432 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
433 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
434
435 DISPLAYLEVEL(3, "Using 4 MB block : ");
436 prefs.frameInfo.blockSizeID = LZ4F_max4MB;
437 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
438 { size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs);
439 DISPLAYLEVEL(4, "dstCapacity = %u ; ", (U32)dstCapacity)
440 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs) );
441 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
442 }
443
444 DISPLAYLEVEL(3, "without frame checksum : ");
445 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
446 { size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs);
447 DISPLAYLEVEL(4, "dstCapacity = %u ; ", (U32)dstCapacity)
448 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs) );
449 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
450 }
451
452 DISPLAYLEVEL(3, "LZ4F_compressFrame with block checksum : ");
453 memset(&prefs, 0, sizeof(prefs));
454 prefs.frameInfo.blockChecksumFlag = LZ4F_blockChecksumEnabled;
455 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
456 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
457
458 DISPLAYLEVEL(3, "Decompress with block checksum : ");
459 { size_t iSize = cSize;
460 size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
461 LZ4F_decompressionContext_t dctx;
462 CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
463 CHECK( LZ4F_decompress(dctx, decodedBuffer, &decodedSize, compressedBuffer, &iSize, NULL) );
464 if (decodedSize != testSize) goto _output_error;
465 if (iSize != cSize) goto _output_error;
466 { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1);
467 U64 const crcSrc = XXH64(CNBuffer, testSize, 1);
468 if (crcDest != crcSrc) goto _output_error;
469 }
470 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
471
472 CHECK( LZ4F_freeDecompressionContext(dctx) );
473 }
474
475 /* frame content size tests */
476 { size_t cErr;
477 BYTE* const ostart = (BYTE*)compressedBuffer;
478 BYTE* op = ostart;
479 CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
480
481 DISPLAYLEVEL(3, "compress without frameSize : ");
482 memset(&(prefs.frameInfo), 0, sizeof(prefs.frameInfo));
483 CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
484 op += cErr;
485 CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL));
486 op += cErr;
487 CHECK( LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL) );
488 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart));
489
490 DISPLAYLEVEL(3, "compress with frameSize : ");
491 prefs.frameInfo.contentSize = testSize;
492 op = ostart;
493 CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
494 op += cErr;
495 CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL));
496 op += cErr;
497 CHECK( LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL) );
498 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart));
499
500 DISPLAYLEVEL(3, "compress with wrong frameSize : ");
501 prefs.frameInfo.contentSize = testSize+1;
502 op = ostart;
503 CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
504 op += cErr;
505 CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL));
506 op += cErr;
507 cErr = LZ4F_compressEnd(cctx, op, testSize, NULL);
508 if (!LZ4F_isError(cErr)) goto _output_error;
509 DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(cErr));
510
511 CHECK( LZ4F_freeCompressionContext(cctx) );
512 cctx = NULL;
513 }
514
515 /* dictID tests */
516 { size_t cErr;
517 U32 const dictID = 0x99;
518 CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
519
520 DISPLAYLEVEL(3, "insert a dictID : ");
521 memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo));
522 prefs.frameInfo.dictID = dictID;
523 CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
524 DISPLAYLEVEL(3, "created frame header of size %i bytes \n", (int)cErr);
525
526 DISPLAYLEVEL(3, "read a dictID : ");
527 CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
528 memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo));
529 CHECK( LZ4F_getFrameInfo(dCtx, &prefs.frameInfo, compressedBuffer, &cErr) );
530 if (prefs.frameInfo.dictID != dictID) goto _output_error;
531 DISPLAYLEVEL(3, "%u \n", (U32)prefs.frameInfo.dictID);
532
533 CHECK( LZ4F_freeDecompressionContext(dCtx) ); dCtx = NULL;
534 CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL;
535 }
536
537 /* Dictionary compression test */
538 { size_t const dictSize = 63 KB;
539 size_t const dstCapacity = LZ4F_compressFrameBound(dictSize, NULL);
540 size_t cSizeNoDict, cSizeWithDict;
541 LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize);
542 if (cdict == NULL) goto _output_error;
543 CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
544
545 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : ");
546 CHECK_V(cSizeNoDict,
547 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
548 CNBuffer, dictSize,
549 NULL, NULL) );
550 DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeNoDict);
551
552 CHECK( LZ4F_freeCompressionContext(cctx) );
553 CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
554 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict : ");
555 CHECK_V(cSizeWithDict,
556 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
557 CNBuffer, dictSize,
558 cdict, NULL) );
559 DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
560 (unsigned)dictSize, (unsigned)cSizeWithDict);
561 if ((LZ4_DISTANCE_MAX > dictSize) && (cSizeWithDict >= cSizeNoDict)) goto _output_error; /* must be more efficient */
562 crcOrig = XXH64(CNBuffer, dictSize, 0);
563
564 DISPLAYLEVEL(3, "LZ4F_decompress_usingDict : ");
565 { LZ4F_dctx* dctx;
566 size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
567 size_t compressedSize = cSizeWithDict;
568 CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
569 CHECK( LZ4F_decompress_usingDict(dctx,
570 decodedBuffer, &decodedSize,
571 compressedBuffer, &compressedSize,
572 CNBuffer, dictSize,
573 NULL) );
574 if (compressedSize != cSizeWithDict) goto _output_error;
575 if (decodedSize != dictSize) goto _output_error;
576 { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0);
577 if (crcDest != crcOrig) goto _output_error; }
578 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
579 CHECK( LZ4F_freeDecompressionContext(dctx) );
580 }
581
582 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict, negative level : ");
583 { size_t cSizeLevelMax;
584 LZ4F_preferences_t cParams;
585 memset(&cParams, 0, sizeof(cParams));
586 cParams.compressionLevel = -3;
587 CHECK_V(cSizeLevelMax,
588 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
589 CNBuffer, dictSize,
590 cdict, &cParams) );
591 DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax);
592 }
593
594 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict, level max : ");
595 { size_t cSizeLevelMax;
596 LZ4F_preferences_t cParams;
597 memset(&cParams, 0, sizeof(cParams));
598 cParams.compressionLevel = LZ4F_compressionLevel_max();
599 CHECK_V(cSizeLevelMax,
600 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
601 CNBuffer, dictSize,
602 cdict, &cParams) );
603 DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax);
604 }
605
606 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, multiple linked blocks : ");
607 { size_t cSizeContiguous;
608 size_t const inSize = dictSize * 3;
609 size_t const outCapacity = LZ4F_compressFrameBound(inSize, NULL);
610 LZ4F_preferences_t cParams;
611 memset(&cParams, 0, sizeof(cParams));
612 cParams.frameInfo.blockMode = LZ4F_blockLinked;
613 cParams.frameInfo.blockSizeID = LZ4F_max64KB;
614 CHECK_V(cSizeContiguous,
615 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, outCapacity,
616 CNBuffer, inSize,
617 cdict, &cParams) );
618 DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
619 (unsigned)inSize, (unsigned)cSizeContiguous);
620
621 DISPLAYLEVEL(3, "LZ4F_decompress_usingDict on multiple linked blocks : ");
622 { LZ4F_dctx* dctx;
623 size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
624 size_t compressedSize = cSizeContiguous;
625 CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
626 CHECK( LZ4F_decompress_usingDict(dctx,
627 decodedBuffer, &decodedSize,
628 compressedBuffer, &compressedSize,
629 CNBuffer, dictSize,
630 NULL) );
631 if (compressedSize != cSizeContiguous) goto _output_error;
632 if (decodedSize != inSize) goto _output_error;
633 crcOrig = XXH64(CNBuffer, inSize, 0);
634 { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0);
635 if (crcDest != crcOrig) goto _output_error; }
636 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
637 CHECK( LZ4F_freeDecompressionContext(dctx) );
638 }
639 }
640
641
642 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, multiple independent blocks : ");
643 { size_t cSizeIndep;
644 size_t const inSize = dictSize * 3;
645 size_t const outCapacity = LZ4F_compressFrameBound(inSize, NULL);
646 LZ4F_preferences_t cParams;
647 memset(&cParams, 0, sizeof(cParams));
648 cParams.frameInfo.blockMode = LZ4F_blockIndependent;
649 cParams.frameInfo.blockSizeID = LZ4F_max64KB;
650 CHECK_V(cSizeIndep,
651 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, outCapacity,
652 CNBuffer, inSize,
653 cdict, &cParams) );
654 DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
655 (unsigned)inSize, (unsigned)cSizeIndep);
656
657 DISPLAYLEVEL(3, "LZ4F_decompress_usingDict on multiple independent blocks : ");
658 { LZ4F_dctx* dctx;
659 size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
660 size_t compressedSize = cSizeIndep;
661 CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
662 CHECK( LZ4F_decompress_usingDict(dctx,
663 decodedBuffer, &decodedSize,
664 compressedBuffer, &compressedSize,
665 CNBuffer, dictSize,
666 NULL) );
667 if (compressedSize != cSizeIndep) goto _output_error;
668 if (decodedSize != inSize) goto _output_error;
669 crcOrig = XXH64(CNBuffer, inSize, 0);
670 { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0);
671 if (crcDest != crcOrig) goto _output_error; }
672 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
673 CHECK( LZ4F_freeDecompressionContext(dctx) );
674 }
675 }
676
677 LZ4F_freeCDict(cdict);
678 CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL;
679 }
680
681 DISPLAYLEVEL(3, "getBlockSize test: \n");
682 { size_t result;
683 unsigned blockSizeID;
684 for (blockSizeID = 4; blockSizeID < 8; ++blockSizeID) {
685 result = LZ4F_getBlockSize(blockSizeID);
686 CHECK(result);
687 DISPLAYLEVEL(3, "Returned block size of %u bytes for blockID %u \n",
688 (unsigned)result, blockSizeID);
689 }
690
691 /* Test an invalid input that's too large */
692 result = LZ4F_getBlockSize(8);
693 if(!LZ4F_isError(result) ||
694 LZ4F_getErrorCode(result) != LZ4F_ERROR_maxBlockSize_invalid)
695 goto _output_error;
696
697 /* Test an invalid input that's too small */
698 result = LZ4F_getBlockSize(3);
699 if(!LZ4F_isError(result) ||
700 LZ4F_getErrorCode(result) != LZ4F_ERROR_maxBlockSize_invalid)
701 goto _output_error;
702 }
703
704
705 DISPLAYLEVEL(3, "Skippable frame test : \n");
706 { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
707 unsigned maxBits = FUZ_highbit((U32)decodedBufferSize);
708 BYTE* op = (BYTE*)decodedBuffer;
709 BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
710 BYTE* ip = (BYTE*)compressedBuffer;
711 BYTE* iend = (BYTE*)compressedBuffer + cSize + 8;
712
713 CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
714
715 /* generate skippable frame */
716 FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START);
717 FUZ_writeLE32(ip+4, (U32)cSize);
718
719 DISPLAYLEVEL(3, "random segment sizes : \n");
720 while (ip < iend) {
721 unsigned nbBits = FUZ_rand(&randState) % maxBits;
722 size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
723 size_t oSize = (size_t)(oend-op);
724 if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip);
725 CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
726 op += oSize;
727 ip += iSize;
728 }
729 DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)decodedBufferSize);
730
731 /* generate zero-size skippable frame */
732 DISPLAYLEVEL(3, "zero-size skippable frame\n");
733 ip = (BYTE*)compressedBuffer;
734 op = (BYTE*)decodedBuffer;
735 FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+1);
736 FUZ_writeLE32(ip+4, 0);
737 iend = ip+8;
738
739 while (ip < iend) {
740 unsigned const nbBits = FUZ_rand(&randState) % maxBits;
741 size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
742 size_t oSize = (size_t)(oend-op);
743 if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip);
744 CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
745 op += oSize;
746 ip += iSize;
747 }
748 DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8));
749
750 DISPLAYLEVEL(3, "Skippable frame header complete in first call \n");
751 ip = (BYTE*)compressedBuffer;
752 op = (BYTE*)decodedBuffer;
753 FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+2);
754 FUZ_writeLE32(ip+4, 10);
755 iend = ip+18;
756 while (ip < iend) {
757 size_t iSize = 10;
758 size_t oSize = 10;
759 if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip);
760 CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
761 op += oSize;
762 ip += iSize;
763 }
764 DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8));
765 }
766
767 DISPLAY("Basic tests completed \n");
768 _end:
769 free(CNBuffer);
770 free(compressedBuffer);
771 free(decodedBuffer);
772 LZ4F_freeDecompressionContext(dCtx); dCtx = NULL;
773 LZ4F_freeCompressionContext(cctx); cctx = NULL;
774 return basicTests_error;
775
776 _output_error:
777 basicTests_error = 1;
778 DISPLAY("Error detected ! \n");
779 goto _end;
780 }
781
782
783 typedef enum { o_contiguous, o_noncontiguous, o_overwrite } o_scenario_e;
784
locateBuffDiff(const void * buff1,const void * buff2,size_t size,o_scenario_e o_scenario)785 static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, o_scenario_e o_scenario)
786 {
787 if (displayLevel >= 2) {
788 size_t p=0;
789 const BYTE* b1=(const BYTE*)buff1;
790 const BYTE* b2=(const BYTE*)buff2;
791 DISPLAY("locateBuffDiff: looking for error position \n");
792 if (o_scenario != o_contiguous) {
793 DISPLAY("mode %i: non-contiguous output (%u bytes), cannot search \n",
794 (int)o_scenario, (unsigned)size);
795 return;
796 }
797 while (p < size && b1[p]==b2[p]) p++;
798 if (p != size) {
799 DISPLAY("Error at pos %i/%i : %02X != %02X \n", (int)p, (int)size, b1[p], b2[p]);
800 }
801 }
802 }
803
804 # define EXIT_MSG(...) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
805 DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); exit(1); }
806 # undef CHECK
807 # define CHECK(cond, ...) { if (cond) { EXIT_MSG(__VA_ARGS__); } }
808
809
test_lz4f_decompression_wBuffers(const void * cSrc,size_t cSize,void * dst,size_t dstCapacity,o_scenario_e o_scenario,const void * srcRef,size_t decompressedSize,U64 crcOrig,U32 * const randState,LZ4F_dctx * const dCtx,U32 seed,U32 testNb,int findErrorPos)810 size_t test_lz4f_decompression_wBuffers(
811 const void* cSrc, size_t cSize,
812 void* dst, size_t dstCapacity, o_scenario_e o_scenario,
813 const void* srcRef, size_t decompressedSize,
814 U64 crcOrig,
815 U32* const randState,
816 LZ4F_dctx* const dCtx,
817 U32 seed, U32 testNb,
818 int findErrorPos)
819 {
820 const BYTE* ip = (const BYTE*)cSrc;
821 const BYTE* const iend = ip + cSize;
822
823 BYTE* op = (BYTE*)dst;
824 BYTE* const oend = op + dstCapacity;
825
826 unsigned const suggestedBits = FUZ_highbit((U32)cSize);
827 unsigned const maxBits = MAX(3, suggestedBits);
828 size_t totalOut = 0;
829 size_t moreToFlush = 0;
830 XXH64_state_t xxh64;
831 XXH64_reset(&xxh64, 1);
832 assert(ip < iend);
833 while (ip < iend) {
834 unsigned const nbBitsI = (FUZ_rand(randState) % (maxBits-1)) + 1;
835 unsigned const nbBitsO = (FUZ_rand(randState) % (maxBits)) + 1;
836 size_t const iSizeCand = (FUZ_rand(randState) & ((1<<nbBitsI)-1)) + 1;
837 size_t const iSizeMax = MIN(iSizeCand, (size_t)(iend-ip));
838 size_t iSize = iSizeMax;
839 size_t const oSizeCand = (FUZ_rand(randState) & ((1<<nbBitsO)-1)) + 2;
840 size_t const oSizeMax = MIN(oSizeCand, (size_t)(oend-op));
841 int const sentinelTest = (op + oSizeMax < oend);
842 size_t oSize = oSizeMax;
843 BYTE const mark = (BYTE)(FUZ_rand(randState) & 255);
844 LZ4F_decompressOptions_t dOptions;
845 memset(&dOptions, 0, sizeof(dOptions));
846 dOptions.stableDst = FUZ_rand(randState) & 1;
847 if (o_scenario == o_overwrite) dOptions.stableDst = 0; /* overwrite mode */
848 if (sentinelTest) op[oSizeMax] = mark;
849
850 DISPLAYLEVEL(7, "dstCapacity=%u, presentedInput=%u \n", (unsigned)oSize, (unsigned)iSize);
851
852 /* read data from byte-exact buffer to catch out-of-bound reads */
853 { void* const iBuffer = malloc(iSizeMax);
854 void* const tmpop = (FUZ_rand(randState) & (oSize == 0)) ? NULL : op;
855 const void* const tmpip = (FUZ_rand(randState) & (iSize == 0)) ? NULL : iBuffer;
856 assert(iBuffer != NULL);
857 memcpy(iBuffer, ip, iSizeMax);
858 moreToFlush = LZ4F_decompress(dCtx, tmpop, &oSize, tmpip, &iSize, &dOptions);
859 free(iBuffer);
860 }
861 DISPLAYLEVEL(7, "oSize=%u, readSize=%u \n", (unsigned)oSize, (unsigned)iSize);
862
863 if (sentinelTest) {
864 CHECK(op[oSizeMax] != mark, "op[oSizeMax] = %02X != %02X : "
865 "Decompression overwrites beyond assigned dst size",
866 op[oSizeMax], mark);
867 }
868 if (LZ4F_getErrorCode(moreToFlush) == LZ4F_ERROR_contentChecksum_invalid) {
869 if (findErrorPos) DISPLAYLEVEL(2, "checksum error detected \n");
870 if (findErrorPos) locateBuffDiff(srcRef, dst, decompressedSize, o_scenario);
871 }
872 if (LZ4F_isError(moreToFlush)) return moreToFlush;
873
874 XXH64_update(&xxh64, op, oSize);
875 totalOut += oSize;
876 op += oSize;
877 ip += iSize;
878 if (o_scenario == o_noncontiguous) {
879 if (op == oend) return LZ4F_ERROR_GENERIC; /* can theoretically happen with bogus data */
880 op++; /* create a gap between consecutive output */
881 }
882 if (o_scenario==o_overwrite) op = (BYTE*)dst; /* overwrite destination */
883 if ( (op == oend) /* no more room for output; can happen with bogus input */
884 && (iSize == 0)) /* no input consumed */
885 break;
886 }
887 if (moreToFlush != 0) return LZ4F_ERROR_decompressionFailed;
888 if (totalOut) { /* otherwise, it's a skippable frame */
889 U64 const crcDecoded = XXH64_digest(&xxh64);
890 if (crcDecoded != crcOrig) {
891 if (findErrorPos) locateBuffDiff(srcRef, dst, decompressedSize, o_scenario);
892 return LZ4F_ERROR_contentChecksum_invalid;
893 } }
894 return 0;
895 }
896
897
test_lz4f_decompression(const void * cSrc,size_t cSize,const void * srcRef,size_t decompressedSize,U64 crcOrig,U32 * const randState,LZ4F_dctx * const dCtx,U32 seed,U32 testNb,int findErrorPos)898 size_t test_lz4f_decompression(const void* cSrc, size_t cSize,
899 const void* srcRef, size_t decompressedSize,
900 U64 crcOrig,
901 U32* const randState,
902 LZ4F_dctx* const dCtx,
903 U32 seed, U32 testNb,
904 int findErrorPos)
905 {
906 o_scenario_e const o_scenario = (o_scenario_e)(FUZ_rand(randState) % 3); /* 0 : contiguous; 1 : non-contiguous; 2 : dst overwritten */
907 /* tighten dst buffer conditions */
908 size_t const dstCapacity = (o_scenario == o_noncontiguous) ?
909 (decompressedSize * 2) + 128 :
910 decompressedSize;
911 size_t result;
912 void* const dstBuffer = malloc(dstCapacity);
913 assert(dstBuffer != NULL);
914
915 result = test_lz4f_decompression_wBuffers(cSrc, cSize,
916 dstBuffer, dstCapacity, o_scenario,
917 srcRef, decompressedSize,
918 crcOrig,
919 randState,
920 dCtx,
921 seed, testNb, findErrorPos);
922
923 free(dstBuffer);
924 return result;
925 }
926
927
fuzzerTests(U32 seed,unsigned nbTests,unsigned startTest,double compressibility,U32 duration_s)928 int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration_s)
929 {
930 unsigned testNb = 0;
931 size_t const CNBufferLength = 9 MB; /* needs to be > 2x4MB to test large blocks */
932 void* CNBuffer = NULL;
933 size_t const compressedBufferSize = LZ4F_compressFrameBound(CNBufferLength, NULL) + 4 MB; /* needs some margin */
934 void* compressedBuffer = NULL;
935 void* decodedBuffer = NULL;
936 U32 coreRand = seed;
937 LZ4F_decompressionContext_t dCtx = NULL;
938 LZ4F_decompressionContext_t dCtxNoise = NULL;
939 LZ4F_compressionContext_t cCtx = NULL;
940 clock_t const startClock = clock();
941 clock_t const clockDuration = duration_s * CLOCKS_PER_SEC;
942
943 /* Create buffers */
944 { size_t const creationStatus = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
945 CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); }
946 { size_t const creationStatus = LZ4F_createDecompressionContext(&dCtxNoise, LZ4F_VERSION);
947 CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); }
948 { size_t const creationStatus = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION);
949 CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); }
950 CNBuffer = malloc(CNBufferLength);
951 CHECK(CNBuffer==NULL, "CNBuffer Allocation failed");
952 compressedBuffer = malloc(compressedBufferSize);
953 CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed");
954 decodedBuffer = calloc(1, CNBufferLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */
955 CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed");
956 FUZ_fillCompressibleNoiseBuffer(CNBuffer, CNBufferLength, compressibility, &coreRand);
957
958 /* jump to requested testNb */
959 for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); /* sync randomizer */
960
961 /* main fuzzer test loop */
962 for ( ; (testNb < nbTests) || (clockDuration > FUZ_GetClockSpan(startClock)) ; testNb++) {
963 U32 randState = coreRand ^ prime1;
964 unsigned const srcBits = (FUZ_rand(&randState) % (FUZ_highbit((U32)(CNBufferLength-1)) - 1)) + 1;
965 size_t const srcSize = (FUZ_rand(&randState) & ((1<<srcBits)-1)) + 1;
966 size_t const srcStartId = FUZ_rand(&randState) % (CNBufferLength - srcSize);
967 const BYTE* const srcStart = (const BYTE*)CNBuffer + srcStartId;
968 unsigned const neverFlush = (FUZ_rand(&randState) & 15) == 1;
969 U64 const crcOrig = XXH64(srcStart, srcSize, 1);
970 LZ4F_preferences_t prefs;
971 const LZ4F_preferences_t* prefsPtr = &prefs;
972 size_t cSize;
973
974 (void)FUZ_rand(&coreRand); /* update seed */
975 memset(&prefs, 0, sizeof(prefs));
976 prefs.frameInfo.blockMode = (LZ4F_blockMode_t)(FUZ_rand(&randState) & 1);
977 prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)(4 + (FUZ_rand(&randState) & 3));
978 prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)(FUZ_rand(&randState) & 1);
979 prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)(FUZ_rand(&randState) & 1);
980 prefs.frameInfo.contentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0;
981 prefs.autoFlush = neverFlush ? 0 : (FUZ_rand(&randState) & 7) == 2;
982 prefs.compressionLevel = -5 + (int)(FUZ_rand(&randState) % 11);
983 if ((FUZ_rand(&randState) & 0xF) == 1) prefsPtr = NULL;
984
985 DISPLAYUPDATE(2, "\r%5u ", testNb);
986
987 if ((FUZ_rand(&randState) & 0xFFF) == 0) {
988 /* create a skippable frame (rare case) */
989 BYTE* op = (BYTE*)compressedBuffer;
990 FUZ_writeLE32(op, LZ4F_MAGIC_SKIPPABLE_START + (FUZ_rand(&randState) & 15));
991 FUZ_writeLE32(op+4, (U32)srcSize);
992 cSize = srcSize+8;
993
994 } else if ((FUZ_rand(&randState) & 0xF) == 2) { /* single pass compression (simple) */
995 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(srcSize, prefsPtr), srcStart, srcSize, prefsPtr);
996 CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize));
997
998 } else { /* multi-segments compression */
999 const BYTE* ip = srcStart;
1000 const BYTE* const iend = srcStart + srcSize;
1001 BYTE* op = (BYTE*)compressedBuffer;
1002 BYTE* const oend = op + (neverFlush ? LZ4F_compressFrameBound(srcSize, prefsPtr) : compressedBufferSize); /* when flushes are possible, can't guarantee a max compressed size */
1003 unsigned const maxBits = FUZ_highbit((U32)srcSize);
1004 LZ4F_compressOptions_t cOptions;
1005 memset(&cOptions, 0, sizeof(cOptions));
1006 { size_t const fhSize = LZ4F_compressBegin(cCtx, op, (size_t)(oend-op), prefsPtr);
1007 CHECK(LZ4F_isError(fhSize), "Compression header failed (error %i)",
1008 (int)fhSize);
1009 op += fhSize;
1010 }
1011 while (ip < iend) {
1012 unsigned const nbBitsSeg = FUZ_rand(&randState) % maxBits;
1013 size_t const sampleMax = (FUZ_rand(&randState) & ((1<<nbBitsSeg)-1)) + 1;
1014 size_t const iSize = MIN(sampleMax, (size_t)(iend-ip));
1015 size_t const oSize = LZ4F_compressBound(iSize, prefsPtr);
1016 size_t flushedSize;
1017 cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1);
1018 DISPLAYLEVEL(6, "Sending %u bytes to compress (stableSrc:%u) \n",
1019 (unsigned)iSize, cOptions.stableSrc);
1020
1021 flushedSize = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions);
1022 CHECK(LZ4F_isError(flushedSize), "Compression failed (error %i : %s)",
1023 (int)flushedSize, LZ4F_getErrorName(flushedSize));
1024 op += flushedSize;
1025 ip += iSize;
1026
1027 { unsigned const forceFlush = neverFlush ? 0 : ((FUZ_rand(&randState) & 3) == 1);
1028 if (forceFlush) {
1029 size_t const flushSize = LZ4F_flush(cCtx, op, (size_t)(oend-op), &cOptions);
1030 DISPLAYLEVEL(6,"flushing %u bytes \n", (unsigned)flushSize);
1031 CHECK(LZ4F_isError(flushSize), "Compression failed (error %i)", (int)flushSize);
1032 op += flushSize;
1033 if ((FUZ_rand(&randState) % 1024) == 3) {
1034 /* add an empty block (requires uncompressed flag) */
1035 op[0] = op[1] = op[2] = 0;
1036 op[3] = 0x80; /* 0x80000000U in little-endian format */
1037 op += 4;
1038 if ((prefsPtr!= NULL) && prefsPtr->frameInfo.blockChecksumFlag) {
1039 U32 const bc32 = XXH32(op, 0, 0);
1040 op[0] = (BYTE)bc32; /* little endian format */
1041 op[1] = (BYTE)(bc32>>8);
1042 op[2] = (BYTE)(bc32>>16);
1043 op[3] = (BYTE)(bc32>>24);
1044 op += 4;
1045 } } } }
1046 } /* while (ip<iend) */
1047 CHECK(op>=oend, "LZ4F_compressFrameBound overflow");
1048 { size_t const dstEndSafeSize = LZ4F_compressBound(0, prefsPtr);
1049 int const tooSmallDstEnd = ((FUZ_rand(&randState) & 31) == 3);
1050 size_t const dstEndTooSmallSize = (FUZ_rand(&randState) % dstEndSafeSize) + 1;
1051 size_t const dstEndSize = tooSmallDstEnd ? dstEndTooSmallSize : dstEndSafeSize;
1052 BYTE const canaryByte = (BYTE)(FUZ_rand(&randState) & 255);
1053 size_t flushedSize;
1054 DISPLAYLEVEL(7,"canaryByte at pos %u / %u \n",
1055 (unsigned)((size_t)(op - (BYTE*)compressedBuffer) + dstEndSize),
1056 (unsigned)compressedBufferSize);
1057 assert(op + dstEndSize < (BYTE*)compressedBuffer + compressedBufferSize);
1058 op[dstEndSize] = canaryByte;
1059 flushedSize = LZ4F_compressEnd(cCtx, op, dstEndSize, &cOptions);
1060 CHECK(op[dstEndSize] != canaryByte, "LZ4F_compressEnd writes beyond dstCapacity !");
1061 if (LZ4F_isError(flushedSize)) {
1062 if (tooSmallDstEnd) /* failure is allowed */ continue;
1063 CHECK(!tooSmallDstEnd, "Compression completion failed (error %i : %s)",
1064 (int)flushedSize, LZ4F_getErrorName(flushedSize));
1065 }
1066 op += flushedSize;
1067 }
1068 cSize = (size_t)(op - (BYTE*)compressedBuffer);
1069 DISPLAYLEVEL(5, "\nCompressed %u bytes into %u \n", (U32)srcSize, (U32)cSize);
1070 }
1071
1072
1073 /* multi-segments decompression */
1074 DISPLAYLEVEL(6, "normal decompression \n");
1075 { size_t result = test_lz4f_decompression(compressedBuffer, cSize, srcStart, srcSize, crcOrig, &randState, dCtx, seed, testNb, 1 /*findError*/ );
1076 CHECK (LZ4F_isError(result), "multi-segment decompression failed (error %i => %s)",
1077 (int)result, LZ4F_getErrorName(result));
1078 }
1079
1080 #if 1
1081 /* insert noise into src */
1082 { U32 const maxNbBits = FUZ_highbit((U32)cSize);
1083 size_t pos = 0;
1084 for (;;) {
1085 /* keep some original src */
1086 { U32 const nbBits = FUZ_rand(&randState) % maxNbBits;
1087 size_t const mask = (1<<nbBits) - 1;
1088 size_t const skipLength = FUZ_rand(&randState) & mask;
1089 pos += skipLength;
1090 }
1091 if (pos >= cSize) break;
1092 /* add noise */
1093 { U32 const nbBitsCodes = FUZ_rand(&randState) % maxNbBits;
1094 U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
1095 size_t const mask = (1<<nbBits) - 1;
1096 size_t const rNoiseLength = (FUZ_rand(&randState) & mask) + 1;
1097 size_t const noiseLength = MIN(rNoiseLength, cSize-pos);
1098 size_t const noiseStart = FUZ_rand(&randState) % (CNBufferLength - noiseLength);
1099 memcpy((BYTE*)compressedBuffer + pos, (const char*)CNBuffer + noiseStart, noiseLength);
1100 pos += noiseLength;
1101 } } }
1102
1103 /* test decompression on noisy src */
1104 DISPLAYLEVEL(6, "noisy decompression \n");
1105 test_lz4f_decompression(compressedBuffer, cSize, srcStart, srcSize, crcOrig, &randState, dCtxNoise, seed, testNb, 0 /*don't search error Pos*/ );
1106 /* note : we don't analyze result here : it probably failed, which is expected.
1107 * The sole purpose is to catch potential out-of-bound reads and writes. */
1108 LZ4F_resetDecompressionContext(dCtxNoise); /* context must be reset after an error */
1109 #endif
1110
1111 } /* for ( ; (testNb < nbTests) ; ) */
1112
1113 DISPLAYLEVEL(2, "\rAll tests completed \n");
1114
1115 LZ4F_freeDecompressionContext(dCtx);
1116 LZ4F_freeDecompressionContext(dCtxNoise);
1117 LZ4F_freeCompressionContext(cCtx);
1118 free(CNBuffer);
1119 free(compressedBuffer);
1120 free(decodedBuffer);
1121
1122 if (use_pause) {
1123 DISPLAY("press enter to finish \n");
1124 (void)getchar();
1125 }
1126 return 0;
1127 }
1128
1129
FUZ_usage(const char * programName)1130 int FUZ_usage(const char* programName)
1131 {
1132 DISPLAY( "Usage :\n");
1133 DISPLAY( " %s [args]\n", programName);
1134 DISPLAY( "\n");
1135 DISPLAY( "Arguments :\n");
1136 DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault);
1137 DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n");
1138 DISPLAY( " -s# : Select seed (default:prompt user)\n");
1139 DISPLAY( " -t# : Select starting test number (default:0)\n");
1140 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
1141 DISPLAY( " -v : verbose\n");
1142 DISPLAY( " -h : display help and exit\n");
1143 return 0;
1144 }
1145
1146
main(int argc,const char ** argv)1147 int main(int argc, const char** argv)
1148 {
1149 U32 seed=0;
1150 int seedset=0;
1151 int argNb;
1152 unsigned nbTests = nbTestsDefault;
1153 unsigned testNb = 0;
1154 int proba = FUZ_COMPRESSIBILITY_DEFAULT;
1155 int result=0;
1156 U32 duration=0;
1157 const char* const programName = argv[0];
1158
1159 /* Check command line */
1160 for (argNb=1; argNb<argc; argNb++) {
1161 const char* argument = argv[argNb];
1162
1163 if(!argument) continue; /* Protection if argument empty */
1164
1165 /* Decode command (note : aggregated short commands are allowed) */
1166 if (argument[0]=='-') {
1167 if (!strcmp(argument, "--no-prompt")) {
1168 no_prompt=1;
1169 seedset=1;
1170 displayLevel=1;
1171 continue;
1172 }
1173 argument++;
1174
1175 while (*argument!=0) {
1176 switch(*argument)
1177 {
1178 case 'h':
1179 return FUZ_usage(programName);
1180 case 'v':
1181 argument++;
1182 displayLevel++;
1183 break;
1184 case 'q':
1185 argument++;
1186 displayLevel--;
1187 break;
1188 case 'p': /* pause at the end */
1189 argument++;
1190 use_pause = 1;
1191 break;
1192
1193 case 'i':
1194 argument++;
1195 nbTests=0; duration=0;
1196 while ((*argument>='0') && (*argument<='9')) {
1197 nbTests *= 10;
1198 nbTests += (unsigned)(*argument - '0');
1199 argument++;
1200 }
1201 break;
1202
1203 case 'T':
1204 argument++;
1205 nbTests = 0; duration = 0;
1206 for (;;) {
1207 switch(*argument)
1208 {
1209 case 'm': duration *= 60; argument++; continue;
1210 case 's':
1211 case 'n': argument++; continue;
1212 case '0':
1213 case '1':
1214 case '2':
1215 case '3':
1216 case '4':
1217 case '5':
1218 case '6':
1219 case '7':
1220 case '8':
1221 case '9': duration *= 10; duration += (U32)(*argument++ - '0'); continue;
1222 }
1223 break;
1224 }
1225 break;
1226
1227 case 's':
1228 argument++;
1229 seed=0;
1230 seedset=1;
1231 while ((*argument>='0') && (*argument<='9')) {
1232 seed *= 10;
1233 seed += (U32)(*argument - '0');
1234 argument++;
1235 }
1236 break;
1237 case 't':
1238 argument++;
1239 testNb=0;
1240 while ((*argument>='0') && (*argument<='9')) {
1241 testNb *= 10;
1242 testNb += (unsigned)(*argument - '0');
1243 argument++;
1244 }
1245 break;
1246 case 'P': /* compressibility % */
1247 argument++;
1248 proba=0;
1249 while ((*argument>='0') && (*argument<='9')) {
1250 proba *= 10;
1251 proba += *argument - '0';
1252 argument++;
1253 }
1254 if (proba<0) proba=0;
1255 if (proba>100) proba=100;
1256 break;
1257 default:
1258 ;
1259 return FUZ_usage(programName);
1260 }
1261 }
1262 }
1263 }
1264
1265 /* Get Seed */
1266 DISPLAY("Starting lz4frame tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING);
1267
1268 if (!seedset) {
1269 time_t const t = time(NULL);
1270 U32 const h = XXH32(&t, sizeof(t), 1);
1271 seed = h % 10000;
1272 }
1273 DISPLAY("Seed = %u\n", seed);
1274 if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
1275
1276 nbTests += (nbTests==0); /* avoid zero */
1277
1278 if (testNb==0) result = basicTests(seed, ((double)proba) / 100);
1279 if (result) return 1;
1280 return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, duration);
1281 }
1282
