1 /*
2   LZ4io.c - LZ4 File/Stream Interface
3   Copyright (C) Yann Collet 2011-2020
4 
5   GPL v2 License
6 
7   This program is free software; you can redistribute it and/or modify
8   it under the terms of the GNU General Public License as published by
9   the Free Software Foundation; either version 2 of the License, or
10   (at your option) any later version.
11 
12   This program is distributed in the hope that it will be useful,
13   but WITHOUT ANY WARRANTY; without even the implied warranty of
14   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15   GNU General Public License for more details.
16 
17   You should have received a copy of the GNU General Public License along
18   with this program; if not, write to the Free Software Foundation, Inc.,
19   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 
21   You can contact the author at :
22   - LZ4 source repository : https://github.com/lz4/lz4
23   - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
24 */
25 /*
26   Note : this is stand-alone program.
27   It is not part of LZ4 compression library, it is a user code of the LZ4 library.
28   - The license of LZ4 library is BSD.
29   - The license of xxHash library is BSD.
30   - The license of this source file is GPLv2.
31 */
32 
33 
34 /*-************************************
35 *  Compiler options
36 **************************************/
37 #ifdef _MSC_VER    /* Visual Studio */
38 #  pragma warning(disable : 4127)    /* disable: C4127: conditional expression is constant */
39 #endif
40 #if defined(__MINGW32__) && !defined(_POSIX_SOURCE)
41 #  define _POSIX_SOURCE 1          /* disable %llu warnings with MinGW on Windows */
42 #endif
43 
44 
45 /*****************************
46 *  Includes
47 *****************************/
48 #include "platform.h"  /* Large File Support, SET_BINARY_MODE, SET_SPARSE_FILE_MODE, PLATFORM_POSIX_VERSION, __64BIT__ */
49 #include "util.h"      /* UTIL_getFileStat, UTIL_setFileStat */
50 #include <stdio.h>     /* fprintf, fopen, fread, stdin, stdout, fflush, getchar */
51 #include <stdlib.h>    /* malloc, free */
52 #include <string.h>    /* strerror, strcmp, strlen */
53 #include <time.h>      /* clock */
54 #include <sys/types.h> /* stat64 */
55 #include <sys/stat.h>  /* stat64 */
56 #include "lz4.h"       /* still required for legacy format */
57 #include "lz4hc.h"     /* still required for legacy format */
58 #define LZ4F_STATIC_LINKING_ONLY
59 #include "lz4frame.h"
60 #include "lz4io.h"
61 
62 
63 /*****************************
64 *  Constants
65 *****************************/
66 #define KB *(1 <<10)
67 #define MB *(1 <<20)
68 #define GB *(1U<<30)
69 
70 #define _1BIT  0x01
71 #define _2BITS 0x03
72 #define _3BITS 0x07
73 #define _4BITS 0x0F
74 #define _8BITS 0xFF
75 
76 #define MAGICNUMBER_SIZE    4
77 #define LZ4IO_MAGICNUMBER   0x184D2204
78 #define LZ4IO_SKIPPABLE0    0x184D2A50
79 #define LZ4IO_SKIPPABLEMASK 0xFFFFFFF0
80 #define LEGACY_MAGICNUMBER  0x184C2102
81 
82 #define CACHELINE 64
83 #define LEGACY_BLOCKSIZE   (8 MB)
84 #define MIN_STREAM_BUFSIZE (192 KB)
85 #define LZ4IO_BLOCKSIZEID_DEFAULT 7
86 #define LZ4_MAX_DICT_SIZE (64 KB)
87 
88 
89 /**************************************
90 *  Macros
91 **************************************/
92 #define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
93 #define DISPLAYOUT(...)      fprintf(stdout, __VA_ARGS__)
94 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
95 static int g_displayLevel = 0;   /* 0 : no display  ; 1: errors  ; 2 : + result + interaction + warnings ; 3 : + progression; 4 : + information */
96 
97 #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
98             if ( ((clock() - g_time) > refreshRate)    \
99               || (g_displayLevel>=4) ) {               \
100                 g_time = clock();                      \
101                 DISPLAY(__VA_ARGS__);                  \
102                 if (g_displayLevel>=4) fflush(stderr); \
103         }   }
104 static const clock_t refreshRate = CLOCKS_PER_SEC / 6;
105 static clock_t g_time = 0;
106 #define LZ4IO_STATIC_ASSERT(c)   { enum { LZ4IO_static_assert = 1/(int)(!!(c)) }; }   /* use after variable declarations */
107 
108 
109 /**************************************
110 *  Local Parameters
111 **************************************/
112 
113 struct LZ4IO_prefs_s {
114     int passThrough;
115     int overwrite;
116     int testMode;
117     int blockSizeId;
118     size_t blockSize;
119     int blockChecksum;
120     int streamChecksum;
121     int blockIndependence;
122     int sparseFileSupport;
123     int contentSizeFlag;
124     int useDictionary;
125     unsigned favorDecSpeed;
126     const char* dictionaryFilename;
127     int removeSrcFile;
128 };
129 
130 /**************************************
131 *  Exceptions
132 ***************************************/
133 #ifndef DEBUG
134 #  define DEBUG 0
135 #endif
136 #define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
137 #define EXM_THROW(error, ...)                                             \
138 {                                                                         \
139     DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
140     DISPLAYLEVEL(1, "Error %i : ", error);                                \
141     DISPLAYLEVEL(1, __VA_ARGS__);                                         \
142     DISPLAYLEVEL(1, " \n");                                               \
143     exit(error);                                                          \
144 }
145 
146 
147 /**************************************
148 *  Version modifiers
149 **************************************/
150 #define EXTENDED_ARGUMENTS
151 #define EXTENDED_HELP
152 #define EXTENDED_FORMAT
153 #define DEFAULT_DECOMPRESSOR LZ4IO_decompressLZ4F
154 
155 
156 /* ************************************************** */
157 /* ****************** Parameters ******************** */
158 /* ************************************************** */
159 
LZ4IO_defaultPreferences(void)160 LZ4IO_prefs_t* LZ4IO_defaultPreferences(void)
161 {
162     LZ4IO_prefs_t* const ret = (LZ4IO_prefs_t*)malloc(sizeof(*ret));
163     if (!ret) EXM_THROW(21, "Allocation error : not enough memory");
164     ret->passThrough = 0;
165     ret->overwrite = 1;
166     ret->testMode = 0;
167     ret->blockSizeId = LZ4IO_BLOCKSIZEID_DEFAULT;
168     ret->blockSize = 0;
169     ret->blockChecksum = 0;
170     ret->streamChecksum = 1;
171     ret->blockIndependence = 1;
172     ret->sparseFileSupport = 1;
173     ret->contentSizeFlag = 0;
174     ret->useDictionary = 0;
175     ret->favorDecSpeed = 0;
176     ret->dictionaryFilename = NULL;
177     ret->removeSrcFile = 0;
178     return ret;
179 }
180 
LZ4IO_freePreferences(LZ4IO_prefs_t * prefs)181 void LZ4IO_freePreferences(LZ4IO_prefs_t* prefs)
182 {
183     free(prefs);
184 }
185 
186 
LZ4IO_setDictionaryFilename(LZ4IO_prefs_t * const prefs,const char * dictionaryFilename)187 int LZ4IO_setDictionaryFilename(LZ4IO_prefs_t* const prefs, const char* dictionaryFilename)
188 {
189     prefs->dictionaryFilename = dictionaryFilename;
190     prefs->useDictionary = dictionaryFilename != NULL;
191     return prefs->useDictionary;
192 }
193 
194 /* Default setting : passThrough = 0; return : passThrough mode (0/1) */
LZ4IO_setPassThrough(LZ4IO_prefs_t * const prefs,int yes)195 int LZ4IO_setPassThrough(LZ4IO_prefs_t* const prefs, int yes)
196 {
197    prefs->passThrough = (yes!=0);
198    return prefs->passThrough;
199 }
200 
201 
202 /* Default setting : overwrite = 1; return : overwrite mode (0/1) */
LZ4IO_setOverwrite(LZ4IO_prefs_t * const prefs,int yes)203 int LZ4IO_setOverwrite(LZ4IO_prefs_t* const prefs, int yes)
204 {
205    prefs->overwrite = (yes!=0);
206    return prefs->overwrite;
207 }
208 
209 /* Default setting : testMode = 0; return : testMode (0/1) */
LZ4IO_setTestMode(LZ4IO_prefs_t * const prefs,int yes)210 int LZ4IO_setTestMode(LZ4IO_prefs_t* const prefs, int yes)
211 {
212    prefs->testMode = (yes!=0);
213    return prefs->testMode;
214 }
215 
216 /* blockSizeID : valid values : 4-5-6-7 */
LZ4IO_setBlockSizeID(LZ4IO_prefs_t * const prefs,unsigned bsid)217 size_t LZ4IO_setBlockSizeID(LZ4IO_prefs_t* const prefs, unsigned bsid)
218 {
219     static const size_t blockSizeTable[] = { 64 KB, 256 KB, 1 MB, 4 MB };
220     static const unsigned minBlockSizeID = 4;
221     static const unsigned maxBlockSizeID = 7;
222     if ((bsid < minBlockSizeID) || (bsid > maxBlockSizeID)) return 0;
223     prefs->blockSizeId = (int)bsid;
224     prefs->blockSize = blockSizeTable[(unsigned)prefs->blockSizeId-minBlockSizeID];
225     return prefs->blockSize;
226 }
227 
LZ4IO_setBlockSize(LZ4IO_prefs_t * const prefs,size_t blockSize)228 size_t LZ4IO_setBlockSize(LZ4IO_prefs_t* const prefs, size_t blockSize)
229 {
230     static const size_t minBlockSize = 32;
231     static const size_t maxBlockSize = 4 MB;
232     unsigned bsid = 0;
233     if (blockSize < minBlockSize) blockSize = minBlockSize;
234     if (blockSize > maxBlockSize) blockSize = maxBlockSize;
235     prefs->blockSize = blockSize;
236     blockSize--;
237     /* find which of { 64k, 256k, 1MB, 4MB } is closest to blockSize */
238     while (blockSize >>= 2)
239         bsid++;
240     if (bsid < 7) bsid = 7;
241     prefs->blockSizeId = (int)(bsid-3);
242     return prefs->blockSize;
243 }
244 
245 /* Default setting : 1 == independent blocks */
LZ4IO_setBlockMode(LZ4IO_prefs_t * const prefs,LZ4IO_blockMode_t blockMode)246 int LZ4IO_setBlockMode(LZ4IO_prefs_t* const prefs, LZ4IO_blockMode_t blockMode)
247 {
248     prefs->blockIndependence = (blockMode == LZ4IO_blockIndependent);
249     return prefs->blockIndependence;
250 }
251 
252 /* Default setting : 0 == no block checksum */
LZ4IO_setBlockChecksumMode(LZ4IO_prefs_t * const prefs,int enable)253 int LZ4IO_setBlockChecksumMode(LZ4IO_prefs_t* const prefs, int enable)
254 {
255     prefs->blockChecksum = (enable != 0);
256     return prefs->blockChecksum;
257 }
258 
259 /* Default setting : 1 == checksum enabled */
LZ4IO_setStreamChecksumMode(LZ4IO_prefs_t * const prefs,int enable)260 int LZ4IO_setStreamChecksumMode(LZ4IO_prefs_t* const prefs, int enable)
261 {
262     prefs->streamChecksum = (enable != 0);
263     return prefs->streamChecksum;
264 }
265 
266 /* Default setting : 0 (no notification) */
LZ4IO_setNotificationLevel(int level)267 int LZ4IO_setNotificationLevel(int level)
268 {
269     g_displayLevel = level;
270     return g_displayLevel;
271 }
272 
273 /* Default setting : 1 (auto: enabled on file, disabled on stdout) */
LZ4IO_setSparseFile(LZ4IO_prefs_t * const prefs,int enable)274 int LZ4IO_setSparseFile(LZ4IO_prefs_t* const prefs, int enable)
275 {
276     prefs->sparseFileSupport = 2*(enable!=0);  /* 2==force enable */
277     return prefs->sparseFileSupport;
278 }
279 
280 /* Default setting : 0 (disabled) */
LZ4IO_setContentSize(LZ4IO_prefs_t * const prefs,int enable)281 int LZ4IO_setContentSize(LZ4IO_prefs_t* const prefs, int enable)
282 {
283     prefs->contentSizeFlag = (enable!=0);
284     return prefs->contentSizeFlag;
285 }
286 
287 /* Default setting : 0 (disabled) */
LZ4IO_favorDecSpeed(LZ4IO_prefs_t * const prefs,int favor)288 void LZ4IO_favorDecSpeed(LZ4IO_prefs_t* const prefs, int favor)
289 {
290     prefs->favorDecSpeed = (favor!=0);
291 }
292 
LZ4IO_setRemoveSrcFile(LZ4IO_prefs_t * const prefs,unsigned flag)293 void LZ4IO_setRemoveSrcFile(LZ4IO_prefs_t* const prefs, unsigned flag)
294 {
295   prefs->removeSrcFile = (flag>0);
296 }
297 
298 
299 
300 /* ************************************************************************ **
301 ** ********************** LZ4 File / Pipe compression ********************* **
302 ** ************************************************************************ */
303 
LZ4IO_isSkippableMagicNumber(unsigned int magic)304 static int LZ4IO_isSkippableMagicNumber(unsigned int magic) {
305     return (magic & LZ4IO_SKIPPABLEMASK) == LZ4IO_SKIPPABLE0;
306 }
307 
308 
309 /** LZ4IO_openSrcFile() :
310  * condition : `srcFileName` must be non-NULL.
311  * @result : FILE* to `dstFileName`, or NULL if it fails */
LZ4IO_openSrcFile(const char * srcFileName)312 static FILE* LZ4IO_openSrcFile(const char* srcFileName)
313 {
314     FILE* f;
315 
316     if (!strcmp (srcFileName, stdinmark)) {
317         DISPLAYLEVEL(4,"Using stdin for input\n");
318         f = stdin;
319         SET_BINARY_MODE(stdin);
320     } else {
321         f = fopen(srcFileName, "rb");
322         if ( f==NULL ) DISPLAYLEVEL(1, "%s: %s \n", srcFileName, strerror(errno));
323     }
324 
325     return f;
326 }
327 
328 /** FIO_openDstFile() :
329  *  prefs is writable, because sparseFileSupport might be updated.
330  *  condition : `dstFileName` must be non-NULL.
331  * @result : FILE* to `dstFileName`, or NULL if it fails */
LZ4IO_openDstFile(const char * dstFileName,const LZ4IO_prefs_t * const prefs)332 static FILE* LZ4IO_openDstFile(const char* dstFileName, const LZ4IO_prefs_t* const prefs)
333 {
334     FILE* f;
335     assert(dstFileName != NULL);
336 
337     if (!strcmp (dstFileName, stdoutmark)) {
338         DISPLAYLEVEL(4, "Using stdout for output \n");
339         f = stdout;
340         SET_BINARY_MODE(stdout);
341         if (prefs->sparseFileSupport==1) {
342             DISPLAYLEVEL(4, "Sparse File Support automatically disabled on stdout ;"
343                             " to force-enable it, add --sparse command \n");
344         }
345     } else {
346         if (!prefs->overwrite && strcmp (dstFileName, nulmark)) {  /* Check if destination file already exists */
347             FILE* const testf = fopen( dstFileName, "rb" );
348             if (testf != NULL) {  /* dest exists, prompt for overwrite authorization */
349                 fclose(testf);
350                 if (g_displayLevel <= 1) {  /* No interaction possible */
351                     DISPLAY("%s already exists; not overwritten  \n", dstFileName);
352                     return NULL;
353                 }
354                 DISPLAY("%s already exists; do you wish to overwrite (y/N) ? ", dstFileName);
355                 {   int ch = getchar();
356                     if ((ch!='Y') && (ch!='y')) {
357                         DISPLAY("    not overwritten  \n");
358                         return NULL;
359                     }
360                     while ((ch!=EOF) && (ch!='\n')) ch = getchar();  /* flush rest of input line */
361         }   }   }
362         f = fopen( dstFileName, "wb" );
363         if (f==NULL) DISPLAYLEVEL(1, "%s: %s\n", dstFileName, strerror(errno));
364     }
365 
366     /* sparse file */
367     {   int const sparseMode = (prefs->sparseFileSupport - (f==stdout)) > 0;
368         if (f && sparseMode) { SET_SPARSE_FILE_MODE(f); }
369     }
370 
371     return f;
372 }
373 
374 
375 
376 /***************************************
377 *   Legacy Compression
378 ***************************************/
379 
380 /* unoptimized version; solves endianess & alignment issues */
LZ4IO_writeLE32(void * p,unsigned value32)381 static void LZ4IO_writeLE32 (void* p, unsigned value32)
382 {
383     unsigned char* const dstPtr = (unsigned char*)p;
384     dstPtr[0] = (unsigned char)value32;
385     dstPtr[1] = (unsigned char)(value32 >> 8);
386     dstPtr[2] = (unsigned char)(value32 >> 16);
387     dstPtr[3] = (unsigned char)(value32 >> 24);
388 }
389 
LZ4IO_LZ4_compress(const char * src,char * dst,int srcSize,int dstSize,int cLevel)390 static int LZ4IO_LZ4_compress(const char* src, char* dst, int srcSize, int dstSize, int cLevel)
391 {
392     (void)cLevel;
393     return LZ4_compress_fast(src, dst, srcSize, dstSize, 1);
394 }
395 
396 /* LZ4IO_compressFilename_Legacy :
397  * This function is intentionally "hidden" (not published in .h)
398  * It generates compressed streams using the old 'legacy' format */
LZ4IO_compressFilename_Legacy(const char * input_filename,const char * output_filename,int compressionlevel,const LZ4IO_prefs_t * prefs)399 int LZ4IO_compressFilename_Legacy(const char* input_filename, const char* output_filename,
400                                   int compressionlevel, const LZ4IO_prefs_t* prefs)
401 {
402     typedef int (*compress_f)(const char* src, char* dst, int srcSize, int dstSize, int cLevel);
403     compress_f const compressionFunction = (compressionlevel < 3) ? LZ4IO_LZ4_compress : LZ4_compress_HC;
404     unsigned long long filesize = 0;
405     unsigned long long compressedfilesize = MAGICNUMBER_SIZE;
406     char* in_buff;
407     char* out_buff;
408     const int outBuffSize = LZ4_compressBound(LEGACY_BLOCKSIZE);
409     FILE* const finput = LZ4IO_openSrcFile(input_filename);
410     FILE* foutput;
411     clock_t clockEnd;
412 
413     /* Init */
414     clock_t const clockStart = clock();
415     if (finput == NULL)
416         EXM_THROW(20, "%s : open file error ", input_filename);
417 
418     foutput = LZ4IO_openDstFile(output_filename, prefs);
419     if (foutput == NULL) {
420         fclose(finput);
421         EXM_THROW(20, "%s : open file error ", input_filename);
422     }
423 
424     /* Allocate Memory */
425     in_buff = (char*)malloc(LEGACY_BLOCKSIZE);
426     out_buff = (char*)malloc((size_t)outBuffSize + 4);
427     if (!in_buff || !out_buff)
428         EXM_THROW(21, "Allocation error : not enough memory");
429 
430     /* Write Archive Header */
431     LZ4IO_writeLE32(out_buff, LEGACY_MAGICNUMBER);
432     if (fwrite(out_buff, 1, MAGICNUMBER_SIZE, foutput) != MAGICNUMBER_SIZE)
433         EXM_THROW(22, "Write error : cannot write header");
434 
435     /* Main Loop */
436     while (1) {
437         int outSize;
438         /* Read Block */
439         size_t const inSize = fread(in_buff, (size_t)1, (size_t)LEGACY_BLOCKSIZE, finput);
440         if (inSize == 0) break;
441         assert(inSize <= LEGACY_BLOCKSIZE);
442         filesize += inSize;
443 
444         /* Compress Block */
445         outSize = compressionFunction(in_buff, out_buff+4, (int)inSize, outBuffSize, compressionlevel);
446         assert(outSize >= 0);
447         compressedfilesize += (unsigned long long)outSize+4;
448         DISPLAYUPDATE(2, "\rRead : %i MB  ==> %.2f%%   ",
449                 (int)(filesize>>20), (double)compressedfilesize/filesize*100);
450 
451         /* Write Block */
452         assert(outSize > 0);
453         assert(outSize < outBuffSize);
454         LZ4IO_writeLE32(out_buff, (unsigned)outSize);
455         if (fwrite(out_buff, 1, (size_t)outSize+4, foutput) != (size_t)(outSize+4)) {
456             EXM_THROW(24, "Write error : cannot write compressed block");
457     }   }
458     if (ferror(finput)) EXM_THROW(25, "Error while reading %s ", input_filename);
459 
460     /* Status */
461     clockEnd = clock();
462     if (clockEnd==clockStart) clockEnd+=1;  /* avoid division by zero (speed) */
463     filesize += !filesize;   /* avoid division by zero (ratio) */
464     DISPLAYLEVEL(2, "\r%79s\r", "");   /* blank line */
465     DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
466         filesize, compressedfilesize, (double)compressedfilesize / filesize * 100);
467     {   double const seconds = (double)(clockEnd - clockStart) / CLOCKS_PER_SEC;
468         DISPLAYLEVEL(4,"Done in %.2f s ==> %.2f MB/s\n", seconds,
469                         (double)filesize / seconds / 1024 / 1024);
470     }
471 
472     /* Close & Free */
473     free(in_buff);
474     free(out_buff);
475     fclose(finput);
476     if (strcmp(output_filename,stdoutmark)) fclose(foutput);   /* do not close stdout */
477 
478     return 0;
479 }
480 
481 #define FNSPACE 30
482 /* LZ4IO_compressMultipleFilenames_Legacy :
483  * This function is intentionally "hidden" (not published in .h)
484  * It generates multiple compressed streams using the old 'legacy' format */
LZ4IO_compressMultipleFilenames_Legacy(const char ** inFileNamesTable,int ifntSize,const char * suffix,int compressionLevel,const LZ4IO_prefs_t * prefs)485 int LZ4IO_compressMultipleFilenames_Legacy(
486                             const char** inFileNamesTable, int ifntSize,
487                             const char* suffix,
488                             int compressionLevel, const LZ4IO_prefs_t* prefs)
489 {
490     int i;
491     int missed_files = 0;
492     char* dstFileName = (char*)malloc(FNSPACE);
493     size_t ofnSize = FNSPACE;
494     const size_t suffixSize = strlen(suffix);
495 
496     if (dstFileName == NULL) return ifntSize;   /* not enough memory */
497 
498     /* loop on each file */
499     for (i=0; i<ifntSize; i++) {
500         size_t const ifnSize = strlen(inFileNamesTable[i]);
501         if (!strcmp(suffix, stdoutmark)) {
502             missed_files += LZ4IO_compressFilename_Legacy(
503                                     inFileNamesTable[i], stdoutmark,
504                                     compressionLevel, prefs);
505             continue;
506         }
507 
508         if (ofnSize <= ifnSize+suffixSize+1) {
509             free(dstFileName);
510             ofnSize = ifnSize + 20;
511             dstFileName = (char*)malloc(ofnSize);
512             if (dstFileName==NULL) {
513                 return ifntSize;
514         }   }
515         strcpy(dstFileName, inFileNamesTable[i]);
516         strcat(dstFileName, suffix);
517 
518         missed_files += LZ4IO_compressFilename_Legacy(
519                                 inFileNamesTable[i], dstFileName,
520                                 compressionLevel, prefs);
521     }
522 
523     /* Close & Free */
524     free(dstFileName);
525 
526     return missed_files;
527 }
528 
529 
530 /*********************************************
531 *  Compression using Frame format
532 *********************************************/
533 
534 typedef struct {
535     void*  srcBuffer;
536     size_t srcBufferSize;
537     void*  dstBuffer;
538     size_t dstBufferSize;
539     LZ4F_compressionContext_t ctx;
540     LZ4F_CDict* cdict;
541 } cRess_t;
542 
LZ4IO_createDict(size_t * dictSize,const char * const dictFilename)543 static void* LZ4IO_createDict(size_t* dictSize, const char* const dictFilename)
544 {
545     size_t readSize;
546     size_t dictEnd = 0;
547     size_t dictLen = 0;
548     size_t dictStart;
549     size_t circularBufSize = LZ4_MAX_DICT_SIZE;
550     char*  circularBuf = (char*)malloc(circularBufSize);
551     char*  dictBuf;
552     FILE* dictFile;
553 
554     if (!circularBuf) EXM_THROW(25, "Allocation error : not enough memory for circular buffer");
555     if (!dictFilename) EXM_THROW(25, "Dictionary error : no filename provided");
556 
557     dictFile = LZ4IO_openSrcFile(dictFilename);
558     if (!dictFile) EXM_THROW(25, "Dictionary error : could not open dictionary file");
559 
560     /* opportunistically seek to the part of the file we care about. If this */
561     /* fails it's not a problem since we'll just read everything anyways.    */
562     if (strcmp(dictFilename, stdinmark)) {
563         (void)UTIL_fseek(dictFile, -LZ4_MAX_DICT_SIZE, SEEK_END);
564     }
565 
566     do {
567         readSize = fread(circularBuf + dictEnd, 1, circularBufSize - dictEnd, dictFile);
568         dictEnd = (dictEnd + readSize) % circularBufSize;
569         dictLen += readSize;
570     } while (readSize>0);
571 
572     if (dictLen > LZ4_MAX_DICT_SIZE) {
573         dictLen = LZ4_MAX_DICT_SIZE;
574     }
575 
576     *dictSize = dictLen;
577 
578     dictStart = (circularBufSize + dictEnd - dictLen) % circularBufSize;
579 
580     if (dictStart == 0) {
581         /* We're in the simple case where the dict starts at the beginning of our circular buffer. */
582         dictBuf = circularBuf;
583         circularBuf = NULL;
584     } else {
585         /* Otherwise, we will alloc a new buffer and copy our dict into that. */
586         dictBuf = (char *)malloc(dictLen ? dictLen : 1);
587         if (!dictBuf) EXM_THROW(25, "Allocation error : not enough memory");
588 
589         memcpy(dictBuf, circularBuf + dictStart, circularBufSize - dictStart);
590         memcpy(dictBuf + circularBufSize - dictStart, circularBuf, dictLen - (circularBufSize - dictStart));
591     }
592 
593     fclose(dictFile);
594     free(circularBuf);
595 
596     return dictBuf;
597 }
598 
LZ4IO_createCDict(const LZ4IO_prefs_t * const prefs)599 static LZ4F_CDict* LZ4IO_createCDict(const LZ4IO_prefs_t* const prefs)
600 {
601     size_t dictionarySize;
602     void* dictionaryBuffer;
603     LZ4F_CDict* cdict;
604     if (!prefs->useDictionary) return NULL;
605     dictionaryBuffer = LZ4IO_createDict(&dictionarySize, prefs->dictionaryFilename);
606     if (!dictionaryBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary");
607     cdict = LZ4F_createCDict(dictionaryBuffer, dictionarySize);
608     free(dictionaryBuffer);
609     return cdict;
610 }
611 
LZ4IO_createCResources(const LZ4IO_prefs_t * const prefs)612 static cRess_t LZ4IO_createCResources(const LZ4IO_prefs_t* const prefs)
613 {
614     const size_t blockSize = prefs->blockSize;
615     cRess_t ress;
616 
617     LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&(ress.ctx), LZ4F_VERSION);
618     if (LZ4F_isError(errorCode)) EXM_THROW(30, "Allocation error : can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
619 
620     /* Allocate Memory */
621     ress.srcBuffer = malloc(blockSize);
622     ress.srcBufferSize = blockSize;
623     ress.dstBufferSize = LZ4F_compressFrameBound(blockSize, NULL);   /* cover worst case */
624     ress.dstBuffer = malloc(ress.dstBufferSize);
625     if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory");
626 
627     ress.cdict = LZ4IO_createCDict(prefs);
628 
629     return ress;
630 }
631 
LZ4IO_freeCResources(cRess_t ress)632 static void LZ4IO_freeCResources(cRess_t ress)
633 {
634     free(ress.srcBuffer);
635     free(ress.dstBuffer);
636 
637     LZ4F_freeCDict(ress.cdict);
638     ress.cdict = NULL;
639 
640     { LZ4F_errorCode_t const errorCode = LZ4F_freeCompressionContext(ress.ctx);
641       if (LZ4F_isError(errorCode)) EXM_THROW(38, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode)); }
642 }
643 
644 /*
645  * LZ4IO_compressFilename_extRess()
646  * result : 0 : compression completed correctly
647  *          1 : missing or pb opening srcFileName
648  */
649 static int
LZ4IO_compressFilename_extRess(cRess_t ress,const char * srcFileName,const char * dstFileName,int compressionLevel,const LZ4IO_prefs_t * const io_prefs)650 LZ4IO_compressFilename_extRess(cRess_t ress,
651                                const char* srcFileName, const char* dstFileName,
652                                int compressionLevel, const LZ4IO_prefs_t* const io_prefs)
653 {
654     unsigned long long filesize = 0;
655     unsigned long long compressedfilesize = 0;
656     FILE* dstFile;
657     void* const srcBuffer = ress.srcBuffer;
658     void* const dstBuffer = ress.dstBuffer;
659     const size_t dstBufferSize = ress.dstBufferSize;
660     const size_t blockSize = io_prefs->blockSize;
661     size_t readSize;
662     LZ4F_compressionContext_t ctx = ress.ctx;   /* just a pointer */
663     LZ4F_preferences_t prefs;
664 
665     /* Init */
666     FILE* const srcFile = LZ4IO_openSrcFile(srcFileName);
667     if (srcFile == NULL) return 1;
668     dstFile = LZ4IO_openDstFile(dstFileName, io_prefs);
669     if (dstFile == NULL) { fclose(srcFile); return 1; }
670     memset(&prefs, 0, sizeof(prefs));
671 
672     /* Set compression parameters */
673     prefs.autoFlush = 1;
674     prefs.compressionLevel = compressionLevel;
675     prefs.frameInfo.blockMode = (LZ4F_blockMode_t)io_prefs->blockIndependence;
676     prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)io_prefs->blockSizeId;
677     prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)io_prefs->blockChecksum;
678     prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)io_prefs->streamChecksum;
679     prefs.favorDecSpeed = io_prefs->favorDecSpeed;
680     if (io_prefs->contentSizeFlag) {
681       U64 const fileSize = UTIL_getOpenFileSize(srcFile);
682       prefs.frameInfo.contentSize = fileSize;   /* == 0 if input == stdin */
683       if (fileSize==0)
684           DISPLAYLEVEL(3, "Warning : cannot determine input content size \n");
685     }
686 
687     /* read first block */
688     readSize  = fread(srcBuffer, (size_t)1, blockSize, srcFile);
689     if (ferror(srcFile)) EXM_THROW(30, "Error reading %s ", srcFileName);
690     filesize += readSize;
691 
692     /* single-block file */
693     if (readSize < blockSize) {
694         /* Compress in single pass */
695         size_t const cSize = LZ4F_compressFrame_usingCDict(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, ress.cdict, &prefs);
696         if (LZ4F_isError(cSize))
697             EXM_THROW(31, "Compression failed : %s", LZ4F_getErrorName(cSize));
698         compressedfilesize = cSize;
699         DISPLAYUPDATE(2, "\rRead : %u MB   ==> %.2f%%   ",
700                       (unsigned)(filesize>>20), (double)compressedfilesize/(filesize+!filesize)*100);   /* avoid division by zero */
701 
702         /* Write Block */
703         if (fwrite(dstBuffer, 1, cSize, dstFile) != cSize) {
704             EXM_THROW(32, "Write error : failed writing single-block compressed frame");
705     }   }
706 
707     else
708 
709     /* multiple-blocks file */
710     {
711         /* Write Frame Header */
712         size_t const headerSize = LZ4F_compressBegin_usingCDict(ctx, dstBuffer, dstBufferSize, ress.cdict, &prefs);
713         if (LZ4F_isError(headerSize)) EXM_THROW(33, "File header generation failed : %s", LZ4F_getErrorName(headerSize));
714         if (fwrite(dstBuffer, 1, headerSize, dstFile) != headerSize)
715             EXM_THROW(34, "Write error : cannot write header");
716         compressedfilesize += headerSize;
717 
718         /* Main Loop - one block at a time */
719         while (readSize>0) {
720             size_t const outSize = LZ4F_compressUpdate(ctx, dstBuffer, dstBufferSize, srcBuffer, readSize, NULL);
721             if (LZ4F_isError(outSize))
722                 EXM_THROW(35, "Compression failed : %s", LZ4F_getErrorName(outSize));
723             compressedfilesize += outSize;
724             DISPLAYUPDATE(2, "\rRead : %u MB   ==> %.2f%%   ",
725                         (unsigned)(filesize>>20), (double)compressedfilesize/filesize*100);
726 
727             /* Write Block */
728             if (fwrite(dstBuffer, 1, outSize, dstFile) != outSize)
729                 EXM_THROW(36, "Write error : cannot write compressed block");
730 
731             /* Read next block */
732             readSize  = fread(srcBuffer, (size_t)1, (size_t)blockSize, srcFile);
733             filesize += readSize;
734         }
735         if (ferror(srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName);
736 
737         /* End of Frame mark */
738         {   size_t const endSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL);
739             if (LZ4F_isError(endSize))
740                 EXM_THROW(38, "End of frame error : %s", LZ4F_getErrorName(endSize));
741             if (fwrite(dstBuffer, 1, endSize, dstFile) != endSize)
742                 EXM_THROW(39, "Write error : cannot write end of frame");
743             compressedfilesize += endSize;
744     }   }
745 
746     /* Release file handlers */
747     fclose (srcFile);
748     if (strcmp(dstFileName,stdoutmark)) fclose (dstFile);  /* do not close stdout */
749 
750     /* Copy owner, file permissions and modification time */
751     {   stat_t statbuf;
752         if (strcmp (srcFileName, stdinmark)
753          && strcmp (dstFileName, stdoutmark)
754          && strcmp (dstFileName, nulmark)
755          && UTIL_getFileStat(srcFileName, &statbuf)) {
756             UTIL_setFileStat(dstFileName, &statbuf);
757     }   }
758 
759     if (io_prefs->removeSrcFile) {  /* remove source file : --rm */
760         if (remove(srcFileName))
761             EXM_THROW(40, "Remove error : %s: %s", srcFileName, strerror(errno));
762     }
763 
764     /* Final Status */
765     DISPLAYLEVEL(2, "\r%79s\r", "");
766     DISPLAYLEVEL(2, "Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
767                     filesize, compressedfilesize,
768                     (double)compressedfilesize / (filesize + !filesize /* avoid division by zero */ ) * 100);
769 
770     return 0;
771 }
772 
773 
LZ4IO_compressFilename(const char * srcFileName,const char * dstFileName,int compressionLevel,const LZ4IO_prefs_t * prefs)774 int LZ4IO_compressFilename(const char* srcFileName, const char* dstFileName, int compressionLevel, const LZ4IO_prefs_t* prefs)
775 {
776     UTIL_time_t const timeStart = UTIL_getTime();
777     clock_t const cpuStart = clock();
778     cRess_t const ress = LZ4IO_createCResources(prefs);
779 
780     int const result = LZ4IO_compressFilename_extRess(ress, srcFileName, dstFileName, compressionLevel, prefs);
781 
782     /* Free resources */
783     LZ4IO_freeCResources(ress);
784 
785     /* Final Status */
786     {   clock_t const cpuEnd = clock();
787         double const cpuLoad_s = (double)(cpuEnd - cpuStart) / CLOCKS_PER_SEC;
788         U64 const timeLength_ns = UTIL_clockSpanNano(timeStart);
789         double const timeLength_s = (double)timeLength_ns / 1000000000;
790         DISPLAYLEVEL(4, "Completed in %.2f sec  (cpu load : %.0f%%)\n",
791                         timeLength_s, (cpuLoad_s / timeLength_s) * 100);
792     }
793 
794     return result;
795 }
796 
797 
LZ4IO_compressMultipleFilenames(const char ** inFileNamesTable,int ifntSize,const char * suffix,int compressionLevel,const LZ4IO_prefs_t * prefs)798 int LZ4IO_compressMultipleFilenames(
799                               const char** inFileNamesTable, int ifntSize,
800                               const char* suffix,
801                               int compressionLevel,
802                               const LZ4IO_prefs_t* prefs)
803 {
804     int i;
805     int missed_files = 0;
806     char* dstFileName = (char*)malloc(FNSPACE);
807     size_t ofnSize = FNSPACE;
808     const size_t suffixSize = strlen(suffix);
809     cRess_t ress;
810 
811     if (dstFileName == NULL) return ifntSize;   /* not enough memory */
812     ress = LZ4IO_createCResources(prefs);
813 
814     /* loop on each file */
815     for (i=0; i<ifntSize; i++) {
816         size_t const ifnSize = strlen(inFileNamesTable[i]);
817         if (!strcmp(suffix, stdoutmark)) {
818             missed_files += LZ4IO_compressFilename_extRess(ress,
819                                     inFileNamesTable[i], stdoutmark,
820                                     compressionLevel, prefs);
821             continue;
822         }
823         if (ofnSize <= ifnSize+suffixSize+1) {
824             free(dstFileName);
825             ofnSize = ifnSize + 20;
826             dstFileName = (char*)malloc(ofnSize);
827             if (dstFileName==NULL) {
828                 LZ4IO_freeCResources(ress);
829                 return ifntSize;
830         }   }
831         strcpy(dstFileName, inFileNamesTable[i]);
832         strcat(dstFileName, suffix);
833 
834         missed_files += LZ4IO_compressFilename_extRess(ress,
835                                 inFileNamesTable[i], dstFileName,
836                                 compressionLevel, prefs);
837     }
838 
839     /* Close & Free */
840     LZ4IO_freeCResources(ress);
841     free(dstFileName);
842 
843     return missed_files;
844 }
845 
846 
847 /* ********************************************************************* */
848 /* ********************** LZ4 file-stream Decompression **************** */
849 /* ********************************************************************* */
850 
851 /* It's presumed that s points to a memory space of size >= 4 */
LZ4IO_readLE32(const void * s)852 static unsigned LZ4IO_readLE32 (const void* s)
853 {
854     const unsigned char* const srcPtr = (const unsigned char*)s;
855     unsigned value32 = srcPtr[0];
856     value32 += (unsigned)srcPtr[1] <<  8;
857     value32 += (unsigned)srcPtr[2] << 16;
858     value32 += (unsigned)srcPtr[3] << 24;
859     return value32;
860 }
861 
862 
863 static unsigned
LZ4IO_fwriteSparse(FILE * file,const void * buffer,size_t bufferSize,int sparseFileSupport,unsigned storedSkips)864 LZ4IO_fwriteSparse(FILE* file,
865                    const void* buffer, size_t bufferSize,
866                    int sparseFileSupport,
867                    unsigned storedSkips)
868 {
869     const size_t sizeT = sizeof(size_t);
870     const size_t maskT = sizeT -1 ;
871     const size_t* const bufferT = (const size_t*)buffer;   /* Buffer is supposed malloc'ed, hence aligned on size_t */
872     const size_t* ptrT = bufferT;
873     size_t bufferSizeT = bufferSize / sizeT;
874     const size_t* const bufferTEnd = bufferT + bufferSizeT;
875     const size_t segmentSizeT = (32 KB) / sizeT;
876     int const sparseMode = (sparseFileSupport - (file==stdout)) > 0;
877 
878     if (!sparseMode) {  /* normal write */
879         size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
880         if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block");
881         return 0;
882     }
883 
884     /* avoid int overflow */
885     if (storedSkips > 1 GB) {
886         int const seekResult = UTIL_fseek(file, 1 GB, SEEK_CUR);
887         if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)");
888         storedSkips -= 1 GB;
889     }
890 
891     while (ptrT < bufferTEnd) {
892         size_t seg0SizeT = segmentSizeT;
893         size_t nb0T;
894 
895         /* count leading zeros */
896         if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT;
897         bufferSizeT -= seg0SizeT;
898         for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ;
899         storedSkips += (unsigned)(nb0T * sizeT);
900 
901         if (nb0T != seg0SizeT) {   /* not all 0s */
902             errno = 0;
903             {   int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR);
904                 if (seekResult) EXM_THROW(72, "Sparse skip error(%d): %s ; try --no-sparse", (int)errno, strerror(errno));
905             }
906             storedSkips = 0;
907             seg0SizeT -= nb0T;
908             ptrT += nb0T;
909             {   size_t const sizeCheck = fwrite(ptrT, sizeT, seg0SizeT, file);
910                 if (sizeCheck != seg0SizeT) EXM_THROW(73, "Write error : cannot write decoded block");
911         }   }
912         ptrT += seg0SizeT;
913     }
914 
915     if (bufferSize & maskT) {  /* size not multiple of sizeT : implies end of block */
916         const char* const restStart = (const char*)bufferTEnd;
917         const char* restPtr = restStart;
918         size_t const restSize =  bufferSize & maskT;
919         const char* const restEnd = restStart + restSize;
920         for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ;
921         storedSkips += (unsigned) (restPtr - restStart);
922         if (restPtr != restEnd) {
923             int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR);
924             if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse");
925             storedSkips = 0;
926             {   size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file);
927                 if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(75, "Write error : cannot write decoded end of block");
928         }   }
929     }
930 
931     return storedSkips;
932 }
933 
LZ4IO_fwriteSparseEnd(FILE * file,unsigned storedSkips)934 static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
935 {
936     if (storedSkips>0) {   /* implies sparseFileSupport>0 */
937         const char lastZeroByte[1] = { 0 };
938         if (UTIL_fseek(file, storedSkips-1, SEEK_CUR) != 0)
939             EXM_THROW(69, "Final skip error (sparse file)\n");
940         if (fwrite(lastZeroByte, 1, 1, file) != 1)
941             EXM_THROW(69, "Write error : cannot write last zero\n");
942     }
943 }
944 
945 
946 static unsigned g_magicRead = 0;   /* out-parameter of LZ4IO_decodeLegacyStream() */
LZ4IO_decodeLegacyStream(FILE * finput,FILE * foutput,const LZ4IO_prefs_t * prefs)947 static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput, const LZ4IO_prefs_t* prefs)
948 {
949     unsigned long long streamSize = 0;
950     unsigned storedSkips = 0;
951 
952     /* Allocate Memory */
953     char* const in_buff  = (char*)malloc((size_t)LZ4_compressBound(LEGACY_BLOCKSIZE));
954     char* const out_buff = (char*)malloc(LEGACY_BLOCKSIZE);
955     if (!in_buff || !out_buff) EXM_THROW(51, "Allocation error : not enough memory");
956 
957     /* Main Loop */
958     while (1) {
959         unsigned int blockSize;
960 
961         /* Block Size */
962         {   size_t const sizeCheck = fread(in_buff, 1, 4, finput);
963             if (sizeCheck == 0) break;                   /* Nothing to read : file read is completed */
964             if (sizeCheck != 4) EXM_THROW(52, "Read error : cannot access block size "); }
965             blockSize = LZ4IO_readLE32(in_buff);       /* Convert to Little Endian */
966             if (blockSize > LZ4_COMPRESSBOUND(LEGACY_BLOCKSIZE)) {
967             /* Cannot read next block : maybe new stream ? */
968             g_magicRead = blockSize;
969             break;
970         }
971 
972         /* Read Block */
973         { size_t const sizeCheck = fread(in_buff, 1, blockSize, finput);
974           if (sizeCheck!=blockSize) EXM_THROW(52, "Read error : cannot access compressed block !"); }
975 
976         /* Decode Block */
977         {   int const decodeSize = LZ4_decompress_safe(in_buff, out_buff, (int)blockSize, LEGACY_BLOCKSIZE);
978             if (decodeSize < 0) EXM_THROW(53, "Decoding Failed ! Corrupted input detected !");
979             streamSize += (unsigned long long)decodeSize;
980             /* Write Block */
981             storedSkips = LZ4IO_fwriteSparse(foutput, out_buff, (size_t)decodeSize, prefs->sparseFileSupport, storedSkips); /* success or die */
982     }   }
983     if (ferror(finput)) EXM_THROW(54, "Read error : ferror");
984 
985     LZ4IO_fwriteSparseEnd(foutput, storedSkips);
986 
987     /* Free */
988     free(in_buff);
989     free(out_buff);
990 
991     return streamSize;
992 }
993 
994 
995 
996 typedef struct {
997     void*  srcBuffer;
998     size_t srcBufferSize;
999     void*  dstBuffer;
1000     size_t dstBufferSize;
1001     FILE*  dstFile;
1002     LZ4F_decompressionContext_t dCtx;
1003     void*  dictBuffer;
1004     size_t dictBufferSize;
1005 } dRess_t;
1006 
LZ4IO_loadDDict(dRess_t * ress,const LZ4IO_prefs_t * const prefs)1007 static void LZ4IO_loadDDict(dRess_t* ress, const LZ4IO_prefs_t* const prefs)
1008 {
1009     if (!prefs->useDictionary) {
1010         ress->dictBuffer = NULL;
1011         ress->dictBufferSize = 0;
1012         return;
1013     }
1014 
1015     ress->dictBuffer = LZ4IO_createDict(&ress->dictBufferSize, prefs->dictionaryFilename);
1016     if (!ress->dictBuffer) EXM_THROW(25, "Dictionary error : could not create dictionary");
1017 }
1018 
1019 static const size_t LZ4IO_dBufferSize = 64 KB;
LZ4IO_createDResources(const LZ4IO_prefs_t * const prefs)1020 static dRess_t LZ4IO_createDResources(const LZ4IO_prefs_t* const prefs)
1021 {
1022     dRess_t ress;
1023 
1024     /* init */
1025     LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&ress.dCtx, LZ4F_VERSION);
1026     if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
1027 
1028     /* Allocate Memory */
1029     ress.srcBufferSize = LZ4IO_dBufferSize;
1030     ress.srcBuffer = malloc(ress.srcBufferSize);
1031     ress.dstBufferSize = LZ4IO_dBufferSize;
1032     ress.dstBuffer = malloc(ress.dstBufferSize);
1033     if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
1034 
1035     LZ4IO_loadDDict(&ress, prefs);
1036 
1037     ress.dstFile = NULL;
1038     return ress;
1039 }
1040 
LZ4IO_freeDResources(dRess_t ress)1041 static void LZ4IO_freeDResources(dRess_t ress)
1042 {
1043     LZ4F_errorCode_t errorCode = LZ4F_freeDecompressionContext(ress.dCtx);
1044     if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
1045     free(ress.srcBuffer);
1046     free(ress.dstBuffer);
1047     free(ress.dictBuffer);
1048 }
1049 
1050 
1051 static unsigned long long
LZ4IO_decompressLZ4F(dRess_t ress,FILE * const srcFile,FILE * const dstFile,const LZ4IO_prefs_t * const prefs)1052 LZ4IO_decompressLZ4F(dRess_t ress,
1053                      FILE* const srcFile, FILE* const dstFile,
1054                      const LZ4IO_prefs_t* const prefs)
1055 {
1056     unsigned long long filesize = 0;
1057     LZ4F_errorCode_t nextToLoad;
1058     unsigned storedSkips = 0;
1059 
1060     /* Init feed with magic number (already consumed from FILE* sFile) */
1061     {   size_t inSize = MAGICNUMBER_SIZE;
1062         size_t outSize= 0;
1063         LZ4IO_writeLE32(ress.srcBuffer, LZ4IO_MAGICNUMBER);
1064         nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, ress.dstBuffer, &outSize, ress.srcBuffer, &inSize, ress.dictBuffer, ress.dictBufferSize, NULL);
1065         if (LZ4F_isError(nextToLoad)) EXM_THROW(62, "Header error : %s", LZ4F_getErrorName(nextToLoad));
1066     }
1067 
1068     /* Main Loop */
1069     for (;nextToLoad;) {
1070         size_t readSize;
1071         size_t pos = 0;
1072         size_t decodedBytes = ress.dstBufferSize;
1073 
1074         /* Read input */
1075         if (nextToLoad > ress.srcBufferSize) nextToLoad = ress.srcBufferSize;
1076         readSize = fread(ress.srcBuffer, 1, nextToLoad, srcFile);
1077         if (!readSize) break;   /* reached end of file or stream */
1078 
1079         while ((pos < readSize) || (decodedBytes == ress.dstBufferSize)) {  /* still to read, or still to flush */
1080             /* Decode Input (at least partially) */
1081             size_t remaining = readSize - pos;
1082             decodedBytes = ress.dstBufferSize;
1083             nextToLoad = LZ4F_decompress_usingDict(ress.dCtx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, ress.dictBuffer, ress.dictBufferSize, NULL);
1084             if (LZ4F_isError(nextToLoad)) EXM_THROW(66, "Decompression error : %s", LZ4F_getErrorName(nextToLoad));
1085             pos += remaining;
1086 
1087             /* Write Block */
1088             if (decodedBytes) {
1089                 if (!prefs->testMode)
1090                     storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, prefs->sparseFileSupport, storedSkips);
1091                 filesize += decodedBytes;
1092                 DISPLAYUPDATE(2, "\rDecompressed : %u MB  ", (unsigned)(filesize>>20));
1093             }
1094 
1095             if (!nextToLoad) break;
1096         }
1097     }
1098     /* can be out because readSize == 0, which could be an fread() error */
1099     if (ferror(srcFile)) EXM_THROW(67, "Read error");
1100 
1101     if (!prefs->testMode) LZ4IO_fwriteSparseEnd(dstFile, storedSkips);
1102     if (nextToLoad!=0) EXM_THROW(68, "Unfinished stream");
1103 
1104     return filesize;
1105 }
1106 
1107 
1108 /* LZ4IO_passThrough:
1109  * just output the same content as input, no decoding.
1110  * This is a capability of zcat, and by extension lz4cat
1111  * MNstore : contain the first MAGICNUMBER_SIZE bytes already read from finput
1112  */
1113 #define PTSIZE  (64 KB)
1114 #define PTSIZET (PTSIZE / sizeof(size_t))
1115 static unsigned long long
LZ4IO_passThrough(FILE * finput,FILE * foutput,unsigned char MNstore[MAGICNUMBER_SIZE],int sparseFileSupport)1116 LZ4IO_passThrough(FILE* finput, FILE* foutput,
1117                   unsigned char MNstore[MAGICNUMBER_SIZE],
1118                   int sparseFileSupport)
1119 {
1120 	size_t buffer[PTSIZET];
1121     size_t readBytes = 1;
1122     unsigned long long total = MAGICNUMBER_SIZE;
1123     unsigned storedSkips = 0;
1124 
1125     if (fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput) != MAGICNUMBER_SIZE) {
1126         EXM_THROW(50, "Pass-through write error");
1127     }
1128     while (readBytes) {
1129         readBytes = fread(buffer, 1, sizeof(buffer), finput);
1130         total += readBytes;
1131         storedSkips = LZ4IO_fwriteSparse(foutput, buffer, readBytes, sparseFileSupport, storedSkips);
1132     }
1133     if (ferror(finput)) EXM_THROW(51, "Read Error");
1134 
1135     LZ4IO_fwriteSparseEnd(foutput, storedSkips);
1136     return total;
1137 }
1138 
1139 
1140 /** Safely handle cases when (unsigned)offset > LONG_MAX */
fseek_u32(FILE * fp,unsigned offset,int where)1141 static int fseek_u32(FILE *fp, unsigned offset, int where)
1142 {
1143     const unsigned stepMax = 1U << 30;
1144     int errorNb = 0;
1145 
1146     if (where != SEEK_CUR) return -1;  /* Only allows SEEK_CUR */
1147     while (offset > 0) {
1148         unsigned s = offset;
1149         if (s > stepMax) s = stepMax;
1150         errorNb = UTIL_fseek(fp, (long) s, SEEK_CUR);
1151         if (errorNb != 0) break;
1152         offset -= s;
1153     }
1154     return errorNb;
1155 }
1156 
1157 #define ENDOFSTREAM ((unsigned long long)-1)
1158 static unsigned long long
selectDecoder(dRess_t ress,FILE * finput,FILE * foutput,const LZ4IO_prefs_t * const prefs)1159 selectDecoder(dRess_t ress,
1160               FILE* finput, FILE* foutput,
1161               const LZ4IO_prefs_t* const prefs)
1162 {
1163     unsigned char MNstore[MAGICNUMBER_SIZE];
1164     unsigned magicNumber;
1165     static unsigned nbFrames = 0;
1166 
1167     /* init */
1168     nbFrames++;
1169 
1170     /* Check Archive Header */
1171     if (g_magicRead) {  /* magic number already read from finput (see legacy frame)*/
1172         magicNumber = g_magicRead;
1173         g_magicRead = 0;
1174     } else {
1175         size_t const nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput);
1176         if (nbReadBytes==0) { nbFrames = 0; return ENDOFSTREAM; }   /* EOF */
1177         if (nbReadBytes != MAGICNUMBER_SIZE)
1178           EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
1179         magicNumber = LZ4IO_readLE32(MNstore);   /* Little Endian format */
1180     }
1181     if (LZ4IO_isSkippableMagicNumber(magicNumber))
1182         magicNumber = LZ4IO_SKIPPABLE0;   /* fold skippable magic numbers */
1183 
1184     switch(magicNumber)
1185     {
1186     case LZ4IO_MAGICNUMBER:
1187         return LZ4IO_decompressLZ4F(ress, finput, foutput, prefs);
1188     case LEGACY_MAGICNUMBER:
1189         DISPLAYLEVEL(4, "Detected : Legacy format \n");
1190         return LZ4IO_decodeLegacyStream(finput, foutput, prefs);
1191     case LZ4IO_SKIPPABLE0:
1192         DISPLAYLEVEL(4, "Skipping detected skippable area \n");
1193         {   size_t const nbReadBytes = fread(MNstore, 1, 4, finput);
1194             if (nbReadBytes != 4)
1195                 EXM_THROW(42, "Stream error : skippable size unreadable");
1196         }
1197         {   unsigned const size = LZ4IO_readLE32(MNstore);
1198             int const errorNb = fseek_u32(finput, size, SEEK_CUR);
1199             if (errorNb != 0)
1200                 EXM_THROW(43, "Stream error : cannot skip skippable area");
1201         }
1202         return 0;
1203     EXTENDED_FORMAT;  /* macro extension for custom formats */
1204     default:
1205         if (nbFrames == 1) {  /* just started */
1206             /* Wrong magic number at the beginning of 1st stream */
1207             if (!prefs->testMode && prefs->overwrite && prefs->passThrough) {
1208                 nbFrames = 0;
1209                 return LZ4IO_passThrough(finput, foutput, MNstore, prefs->sparseFileSupport);
1210             }
1211             EXM_THROW(44,"Unrecognized header : file cannot be decoded");
1212         }
1213         {   long int const position = ftell(finput);  /* only works for files < 2 GB */
1214             DISPLAYLEVEL(2, "Stream followed by undecodable data ");
1215             if (position != -1L)
1216                 DISPLAYLEVEL(2, "at position %i ", (int)position);
1217             DISPLAYLEVEL(2, "\n");
1218         }
1219         return ENDOFSTREAM;
1220     }
1221 }
1222 
1223 
1224 static int
LZ4IO_decompressSrcFile(dRess_t ress,const char * input_filename,const char * output_filename,const LZ4IO_prefs_t * const prefs)1225 LZ4IO_decompressSrcFile(dRess_t ress,
1226                         const char* input_filename, const char* output_filename,
1227                         const LZ4IO_prefs_t* const prefs)
1228 {
1229     FILE* const foutput = ress.dstFile;
1230     unsigned long long filesize = 0;
1231 
1232     /* Init */
1233     FILE* const finput = LZ4IO_openSrcFile(input_filename);
1234     if (finput==NULL) return 1;
1235     assert(foutput != NULL);
1236 
1237     /* Loop over multiple streams */
1238     for ( ; ; ) {  /* endless loop, see break condition */
1239         unsigned long long const decodedSize =
1240                         selectDecoder(ress, finput, foutput, prefs);
1241         if (decodedSize == ENDOFSTREAM) break;
1242         filesize += decodedSize;
1243     }
1244 
1245     /* Close input */
1246     fclose(finput);
1247     if (prefs->removeSrcFile) {  /* --rm */
1248         if (remove(input_filename))
1249             EXM_THROW(45, "Remove error : %s: %s", input_filename, strerror(errno));
1250     }
1251 
1252     /* Final Status */
1253     DISPLAYLEVEL(2, "\r%79s\r", "");
1254     DISPLAYLEVEL(2, "%-20.20s : decoded %llu bytes \n", input_filename, filesize);
1255     (void)output_filename;
1256 
1257     return 0;
1258 }
1259 
1260 
1261 static int
LZ4IO_decompressDstFile(dRess_t ress,const char * input_filename,const char * output_filename,const LZ4IO_prefs_t * const prefs)1262 LZ4IO_decompressDstFile(dRess_t ress,
1263                         const char* input_filename, const char* output_filename,
1264                         const LZ4IO_prefs_t* const prefs)
1265 {
1266     stat_t statbuf;
1267     int stat_result = 0;
1268     FILE* const foutput = LZ4IO_openDstFile(output_filename, prefs);
1269     if (foutput==NULL) return 1;   /* failure */
1270 
1271     if ( strcmp(input_filename, stdinmark)
1272       && UTIL_getFileStat(input_filename, &statbuf))
1273         stat_result = 1;
1274 
1275     ress.dstFile = foutput;
1276     LZ4IO_decompressSrcFile(ress, input_filename, output_filename, prefs);
1277 
1278     fclose(foutput);
1279 
1280     /* Copy owner, file permissions and modification time */
1281     if ( stat_result != 0
1282       && strcmp (output_filename, stdoutmark)
1283       && strcmp (output_filename, nulmark)) {
1284         UTIL_setFileStat(output_filename, &statbuf);
1285         /* should return value be read ? or is silent fail good enough ? */
1286     }
1287 
1288     return 0;
1289 }
1290 
1291 
LZ4IO_decompressFilename(const char * input_filename,const char * output_filename,const LZ4IO_prefs_t * prefs)1292 int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename, const LZ4IO_prefs_t* prefs)
1293 {
1294     dRess_t const ress = LZ4IO_createDResources(prefs);
1295     clock_t const start = clock();
1296 
1297     int const missingFiles = LZ4IO_decompressDstFile(ress, input_filename, output_filename, prefs);
1298 
1299     clock_t const end = clock();
1300     double const seconds = (double)(end - start) / CLOCKS_PER_SEC;
1301     DISPLAYLEVEL(4, "Done in %.2f sec  \n", seconds);
1302 
1303     LZ4IO_freeDResources(ress);
1304     return missingFiles;
1305 }
1306 
1307 
LZ4IO_decompressMultipleFilenames(const char ** inFileNamesTable,int ifntSize,const char * suffix,const LZ4IO_prefs_t * prefs)1308 int LZ4IO_decompressMultipleFilenames(
1309                             const char** inFileNamesTable, int ifntSize,
1310                             const char* suffix,
1311                             const LZ4IO_prefs_t* prefs)
1312 {
1313     int i;
1314     int skippedFiles = 0;
1315     int missingFiles = 0;
1316     char* outFileName = (char*)malloc(FNSPACE);
1317     size_t ofnSize = FNSPACE;
1318     size_t const suffixSize = strlen(suffix);
1319     dRess_t ress = LZ4IO_createDResources(prefs);
1320 
1321     if (outFileName==NULL) EXM_THROW(70, "Memory allocation error");
1322     ress.dstFile = LZ4IO_openDstFile(stdoutmark, prefs);
1323 
1324     for (i=0; i<ifntSize; i++) {
1325         size_t const ifnSize = strlen(inFileNamesTable[i]);
1326         const char* const suffixPtr = inFileNamesTable[i] + ifnSize - suffixSize;
1327         if (!strcmp(suffix, stdoutmark)) {
1328             missingFiles += LZ4IO_decompressSrcFile(ress, inFileNamesTable[i], stdoutmark, prefs);
1329             continue;
1330         }
1331         if (ofnSize <= ifnSize-suffixSize+1) {
1332             free(outFileName);
1333             ofnSize = ifnSize + 20;
1334             outFileName = (char*)malloc(ofnSize);
1335             if (outFileName==NULL) EXM_THROW(71, "Memory allocation error");
1336         }
1337         if (ifnSize <= suffixSize  ||  strcmp(suffixPtr, suffix) != 0) {
1338             DISPLAYLEVEL(1, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]);
1339             skippedFiles++;
1340             continue;
1341         }
1342         memcpy(outFileName, inFileNamesTable[i], ifnSize - suffixSize);
1343         outFileName[ifnSize-suffixSize] = '\0';
1344         missingFiles += LZ4IO_decompressDstFile(ress, inFileNamesTable[i], outFileName, prefs);
1345     }
1346 
1347     LZ4IO_freeDResources(ress);
1348     free(outFileName);
1349     return missingFiles + skippedFiles;
1350 }
1351 
1352 
1353 /* ********************************************************************* */
1354 /* **********************   LZ4 --list command   *********************** */
1355 /* ********************************************************************* */
1356 
1357 typedef enum
1358 {
1359     lz4Frame = 0,
1360     legacyFrame,
1361     skippableFrame
1362 } LZ4IO_frameType_t;
1363 
1364 typedef struct {
1365     LZ4F_frameInfo_t lz4FrameInfo;
1366     LZ4IO_frameType_t frameType;
1367 } LZ4IO_frameInfo_t;
1368 
1369 #define LZ4IO_INIT_FRAMEINFO  { LZ4F_INIT_FRAMEINFO, lz4Frame }
1370 
1371 typedef struct {
1372     const char* fileName;
1373     unsigned long long fileSize;
1374     unsigned long long frameCount;
1375     LZ4IO_frameInfo_t frameSummary;
1376     unsigned short eqFrameTypes;
1377     unsigned short eqBlockTypes;
1378     unsigned short allContentSize;
1379 } LZ4IO_cFileInfo_t;
1380 
1381 #define LZ4IO_INIT_CFILEINFO  { NULL, 0ULL, 0, LZ4IO_INIT_FRAMEINFO, 1, 1, 1 }
1382 
1383 typedef enum { LZ4IO_LZ4F_OK, LZ4IO_format_not_known, LZ4IO_not_a_file } LZ4IO_infoResult;
1384 
1385 static const char * LZ4IO_frameTypeNames[] = {"LZ4Frame", "LegacyFrame", "SkippableFrame" };
1386 
1387 /* Read block headers and skip block data
1388    Return total blocks size for this frame including block headers,
1389    block checksums and content checksums.
1390    returns 0 in case it can't succesfully skip block data.
1391    Assumes SEEK_CUR after frame header.
1392  */
1393 static unsigned long long
LZ4IO_skipBlocksData(FILE * finput,const LZ4F_blockChecksum_t blockChecksumFlag,const LZ4F_contentChecksum_t contentChecksumFlag)1394 LZ4IO_skipBlocksData(FILE* finput,
1395                const LZ4F_blockChecksum_t blockChecksumFlag,
1396                const LZ4F_contentChecksum_t contentChecksumFlag)
1397 {
1398     unsigned char blockInfo[LZ4F_BLOCK_HEADER_SIZE];
1399     unsigned long long totalBlocksSize = 0;
1400     for (;;) {
1401         if (!fread(blockInfo, 1, LZ4F_BLOCK_HEADER_SIZE, finput)) {
1402             if (feof(finput)) return totalBlocksSize;
1403             return 0;
1404         }
1405         totalBlocksSize += LZ4F_BLOCK_HEADER_SIZE;
1406         {   const unsigned long nextCBlockSize = LZ4IO_readLE32(&blockInfo) & 0x7FFFFFFFU;
1407             const unsigned long nextBlock = nextCBlockSize + (blockChecksumFlag * LZ4F_BLOCK_CHECKSUM_SIZE);
1408             if (nextCBlockSize == 0) {
1409                 /* Reached EndMark */
1410                 if (contentChecksumFlag) {
1411                     /* Skip content checksum */
1412                     if (UTIL_fseek(finput, LZ4F_CONTENT_CHECKSUM_SIZE, SEEK_CUR) != 0) {
1413                         return 0;
1414                     }
1415                     totalBlocksSize += LZ4F_CONTENT_CHECKSUM_SIZE;
1416                 }
1417                 break;
1418             }
1419             totalBlocksSize += nextBlock;
1420             /* skip to the next block */
1421             assert(nextBlock < LONG_MAX);
1422             if (UTIL_fseek(finput, (long)nextBlock, SEEK_CUR) != 0) return 0;
1423     }   }
1424     return totalBlocksSize;
1425 }
1426 
1427 /* For legacy frames only.
1428    Read block headers and skip block data.
1429    Return total blocks size for this frame including block headers.
1430    or 0 in case it can't succesfully skip block data.
1431    This works as long as legacy block header size = magic number size.
1432    Assumes SEEK_CUR after frame header.
1433  */
LZ4IO_skipLegacyBlocksData(FILE * finput)1434 static unsigned long long LZ4IO_skipLegacyBlocksData(FILE* finput)
1435 {
1436     unsigned char blockInfo[LZIO_LEGACY_BLOCK_HEADER_SIZE];
1437     unsigned long long totalBlocksSize = 0;
1438     LZ4IO_STATIC_ASSERT(LZIO_LEGACY_BLOCK_HEADER_SIZE == MAGICNUMBER_SIZE);
1439     for (;;) {
1440         if (!fread(blockInfo, 1, LZIO_LEGACY_BLOCK_HEADER_SIZE, finput)) {
1441             if (feof(finput)) return totalBlocksSize;
1442             return 0;
1443         }
1444         {   const unsigned int nextCBlockSize = LZ4IO_readLE32(&blockInfo);
1445             if ( nextCBlockSize == LEGACY_MAGICNUMBER ||
1446                     nextCBlockSize == LZ4IO_MAGICNUMBER ||
1447                     LZ4IO_isSkippableMagicNumber(nextCBlockSize)) {
1448                 /* Rewind back. we want cursor at the begining of next frame.*/
1449                 if (fseek(finput, -LZIO_LEGACY_BLOCK_HEADER_SIZE, SEEK_CUR) != 0) {
1450                     return 0;
1451                 }
1452                 break;
1453             }
1454             totalBlocksSize += LZIO_LEGACY_BLOCK_HEADER_SIZE + nextCBlockSize;
1455             /* skip to the next block */
1456             if (UTIL_fseek(finput, nextCBlockSize, SEEK_CUR) != 0) {
1457                 return 0;
1458     }   }   }
1459     return totalBlocksSize;
1460 }
1461 
1462 /* LZ4IO_blockTypeID:
1463  * return human-readable block type, following command line convention
1464  * buffer : must be a valid memory area of at least 4 bytes */
LZ4IO_blockTypeID(LZ4F_blockSizeID_t sizeID,LZ4F_blockMode_t blockMode,char buffer[4])1465 const char* LZ4IO_blockTypeID(LZ4F_blockSizeID_t sizeID, LZ4F_blockMode_t blockMode, char buffer[4])
1466 {
1467     buffer[0] = 'B';
1468     assert(sizeID >= 4); assert(sizeID <= 7);
1469     buffer[1] = (char)(sizeID + '0');
1470     buffer[2] = (blockMode == LZ4F_blockIndependent) ? 'I' : 'D';
1471     buffer[3] = 0;
1472     return buffer;
1473 }
1474 
1475 /* buffer : must be valid memory area of at least 10 bytes */
LZ4IO_toHuman(long double size,char * buf)1476 static const char* LZ4IO_toHuman(long double size, char *buf)
1477 {
1478     const char units[] = {"\0KMGTPEZY"};
1479     size_t i = 0;
1480     for (; size >= 1024; i++) size /= 1024;
1481     sprintf(buf, "%.2Lf%c", size, units[i]);
1482     return buf;
1483 }
1484 
1485 /* Get filename without path prefix */
LZ4IO_baseName(const char * input_filename)1486 static const char* LZ4IO_baseName(const char* input_filename)
1487 {
1488     const char* b = strrchr(input_filename, '/');
1489     if (!b) b = strrchr(input_filename, '\\');
1490     if (!b) return input_filename;
1491     return b + 1;
1492 }
1493 
1494 /* Report frame/s information (--list) in verbose mode (-v).
1495  * Will populate file info with fileName and frameSummary where applicable.
1496  * - TODO :
1497  *  + report nb of blocks, hence max. possible decompressed size (when not reported in header)
1498  */
1499 static LZ4IO_infoResult
LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t * cfinfo,const char * input_filename)1500 LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filename)
1501 {
1502     LZ4IO_infoResult result = LZ4IO_format_not_known;  /* default result (error) */
1503     unsigned char buffer[LZ4F_HEADER_SIZE_MAX];
1504     FILE* const finput = LZ4IO_openSrcFile(input_filename);
1505 
1506     if (finput == NULL) return LZ4IO_not_a_file;
1507     cfinfo->fileSize = UTIL_getOpenFileSize(finput);
1508 
1509     while (!feof(finput)) {
1510         LZ4IO_frameInfo_t frameInfo = LZ4IO_INIT_FRAMEINFO;
1511         unsigned magicNumber;
1512         /* Get MagicNumber */
1513         {   size_t const nbReadBytes = fread(buffer, 1, MAGICNUMBER_SIZE, finput);
1514             if (nbReadBytes == 0) { break; } /* EOF */
1515             result = LZ4IO_format_not_known;  /* default result (error) */
1516             if (nbReadBytes != MAGICNUMBER_SIZE) {
1517                 EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
1518         }   }
1519         magicNumber = LZ4IO_readLE32(buffer);   /* Little Endian format */
1520         if (LZ4IO_isSkippableMagicNumber(magicNumber))
1521             magicNumber = LZ4IO_SKIPPABLE0;   /* fold skippable magic numbers */
1522 
1523         switch (magicNumber) {
1524         case LZ4IO_MAGICNUMBER:
1525             if (cfinfo->frameSummary.frameType != lz4Frame) cfinfo->eqFrameTypes = 0;
1526             /* Get frame info */
1527             {   const size_t readBytes = fread(buffer + MAGICNUMBER_SIZE, 1, LZ4F_HEADER_SIZE_MIN - MAGICNUMBER_SIZE, finput);
1528                 if (!readBytes || ferror(finput)) EXM_THROW(71, "Error reading %s", input_filename);
1529             }
1530             {   size_t hSize = LZ4F_headerSize(&buffer, LZ4F_HEADER_SIZE_MIN);
1531                 if (LZ4F_isError(hSize)) break;
1532                 if (hSize > (LZ4F_HEADER_SIZE_MIN + MAGICNUMBER_SIZE)) {
1533                     /* We've already read LZ4F_HEADER_SIZE_MIN so read any extra until hSize*/
1534                     const size_t readBytes = fread(buffer + LZ4F_HEADER_SIZE_MIN, 1, hSize - LZ4F_HEADER_SIZE_MIN, finput);
1535                     if (!readBytes || ferror(finput)) EXM_THROW(72, "Error reading %s", input_filename);
1536                 }
1537                 /* Create decompression context */
1538                 {   LZ4F_dctx* dctx;
1539                     if ( LZ4F_isError(LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION)) ) break;
1540                     {   unsigned const frameInfoError = LZ4F_isError(LZ4F_getFrameInfo(dctx, &frameInfo.lz4FrameInfo, buffer, &hSize));
1541                         LZ4F_freeDecompressionContext(dctx);
1542                         if (frameInfoError) break;
1543                         if ((cfinfo->frameSummary.lz4FrameInfo.blockSizeID != frameInfo.lz4FrameInfo.blockSizeID ||
1544                                 cfinfo->frameSummary.lz4FrameInfo.blockMode != frameInfo.lz4FrameInfo.blockMode)
1545                                 && cfinfo->frameCount != 0)
1546                             cfinfo->eqBlockTypes = 0;
1547                         {   const unsigned long long totalBlocksSize = LZ4IO_skipBlocksData(finput,
1548                                     frameInfo.lz4FrameInfo.blockChecksumFlag,
1549                                     frameInfo.lz4FrameInfo.contentChecksumFlag);
1550                             if (totalBlocksSize) {
1551                                 char bTypeBuffer[5];
1552                                 LZ4IO_blockTypeID(frameInfo.lz4FrameInfo.blockSizeID, frameInfo.lz4FrameInfo.blockMode, bTypeBuffer);
1553                                 DISPLAYLEVEL(3, "    %6llu %14s %5s %8s",
1554                                              cfinfo->frameCount + 1,
1555                                              LZ4IO_frameTypeNames[frameInfo.frameType],
1556                                              bTypeBuffer,
1557                                              frameInfo.lz4FrameInfo.contentChecksumFlag ? "XXH32" : "-");
1558                                 if (frameInfo.lz4FrameInfo.contentSize) {
1559                                     {   double const ratio = (double)(totalBlocksSize + hSize) / frameInfo.lz4FrameInfo.contentSize * 100;
1560                                         DISPLAYLEVEL(3, " %20llu %20llu %9.2f%%\n",
1561                                                      totalBlocksSize + hSize,
1562                                                      frameInfo.lz4FrameInfo.contentSize,
1563                                                      ratio);
1564                                     }
1565                                     /* Now we've consumed frameInfo we can use it to store the total contentSize */
1566                                     frameInfo.lz4FrameInfo.contentSize += cfinfo->frameSummary.lz4FrameInfo.contentSize;
1567                                 }
1568                                 else {
1569                                     DISPLAYLEVEL(3, " %20llu %20s %9s \n", totalBlocksSize + hSize, "-", "-");
1570                                     cfinfo->allContentSize = 0;
1571                                 }
1572                                 result = LZ4IO_LZ4F_OK;
1573             }   }   }   }   }
1574             break;
1575         case LEGACY_MAGICNUMBER:
1576             frameInfo.frameType = legacyFrame;
1577             if (cfinfo->frameSummary.frameType != legacyFrame && cfinfo->frameCount != 0) cfinfo->eqFrameTypes = 0;
1578             cfinfo->eqBlockTypes = 0;
1579             cfinfo->allContentSize = 0;
1580             {   const unsigned long long totalBlocksSize = LZ4IO_skipLegacyBlocksData(finput);
1581                 if (totalBlocksSize) {
1582                     DISPLAYLEVEL(3, "    %6llu %14s %5s %8s %20llu %20s %9s\n",
1583                                  cfinfo->frameCount + 1,
1584                                  LZ4IO_frameTypeNames[frameInfo.frameType],
1585                                  "-", "-",
1586                                  totalBlocksSize + 4,
1587                                  "-", "-");
1588                     result = LZ4IO_LZ4F_OK;
1589             }   }
1590             break;
1591         case LZ4IO_SKIPPABLE0:
1592             frameInfo.frameType = skippableFrame;
1593             if (cfinfo->frameSummary.frameType != skippableFrame && cfinfo->frameCount != 0) cfinfo->eqFrameTypes = 0;
1594             cfinfo->eqBlockTypes = 0;
1595             cfinfo->allContentSize = 0;
1596             {   size_t const nbReadBytes = fread(buffer, 1, 4, finput);
1597                 if (nbReadBytes != 4)
1598                     EXM_THROW(42, "Stream error : skippable size unreadable");
1599             }
1600             {   unsigned const size = LZ4IO_readLE32(buffer);
1601                 int const errorNb = fseek_u32(finput, size, SEEK_CUR);
1602                 if (errorNb != 0)
1603                     EXM_THROW(43, "Stream error : cannot skip skippable area");
1604                 DISPLAYLEVEL(3, "    %6llu %14s %5s %8s %20u %20s %9s\n",
1605                              cfinfo->frameCount + 1,
1606                              "SkippableFrame",
1607                              "-", "-", size + 8, "-", "-");
1608 
1609                 result = LZ4IO_LZ4F_OK;
1610             }
1611             break;
1612         default:
1613             {   long int const position = ftell(finput);  /* only works for files < 2 GB */
1614                 DISPLAYLEVEL(3, "Stream followed by undecodable data ");
1615                 if (position != -1L)
1616                     DISPLAYLEVEL(3, "at position %i ", (int)position);
1617                 DISPLAYLEVEL(3, "\n");
1618             }
1619         break;
1620         }
1621         if (result != LZ4IO_LZ4F_OK) break;
1622         cfinfo->frameSummary = frameInfo;
1623         cfinfo->frameCount++;
1624     }  /* while (!feof(finput)) */
1625     fclose(finput);
1626     return result;
1627 }
1628 
1629 
LZ4IO_displayCompressedFilesInfo(const char ** inFileNames,size_t ifnIdx)1630 int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, size_t ifnIdx)
1631 {
1632     int result = 0;
1633     size_t idx = 0;
1634     if (g_displayLevel < 3) {
1635         DISPLAYOUT("%10s %14s %5s %11s %13s %9s   %s\n",
1636                 "Frames", "Type", "Block", "Compressed", "Uncompressed", "Ratio", "Filename");
1637     }
1638     for (; idx < ifnIdx; idx++) {
1639         /* Get file info */
1640         LZ4IO_cFileInfo_t cfinfo = LZ4IO_INIT_CFILEINFO;
1641         cfinfo.fileName = LZ4IO_baseName(inFileNames[idx]);
1642         if (!UTIL_isRegFile(inFileNames[idx])) {
1643             DISPLAYLEVEL(1, "lz4: %s is not a regular file \n", inFileNames[idx]);
1644             return 0;
1645         }
1646         DISPLAYLEVEL(3, "%s(%llu/%llu)\n", cfinfo.fileName, (unsigned long long)idx + 1, (unsigned  long long)ifnIdx);
1647         DISPLAYLEVEL(3, "    %6s %14s %5s %8s %20s %20s %9s\n",
1648                      "Frame", "Type", "Block", "Checksum", "Compressed", "Uncompressed", "Ratio")
1649         {   LZ4IO_infoResult const op_result = LZ4IO_getCompressedFileInfo(&cfinfo, inFileNames[idx]);
1650             if (op_result != LZ4IO_LZ4F_OK) {
1651                 assert(op_result == LZ4IO_format_not_known);
1652                 DISPLAYLEVEL(1, "lz4: %s: File format not recognized \n", inFileNames[idx]);
1653                 return 0;
1654         }   }
1655         DISPLAYLEVEL(3, "\n");
1656         if (g_displayLevel < 3) {
1657             /* Display Summary */
1658             {   char buffers[3][10];
1659                 DISPLAYOUT("%10llu %14s %5s %11s %13s ",
1660                         cfinfo.frameCount,
1661                         cfinfo.eqFrameTypes ? LZ4IO_frameTypeNames[cfinfo.frameSummary.frameType] : "-" ,
1662                         cfinfo.eqBlockTypes ? LZ4IO_blockTypeID(cfinfo.frameSummary.lz4FrameInfo.blockSizeID,
1663                                                                 cfinfo.frameSummary.lz4FrameInfo.blockMode, buffers[0]) : "-",
1664                         LZ4IO_toHuman((long double)cfinfo.fileSize, buffers[1]),
1665                         cfinfo.allContentSize ? LZ4IO_toHuman((long double)cfinfo.frameSummary.lz4FrameInfo.contentSize, buffers[2]) : "-");
1666                 if (cfinfo.allContentSize) {
1667                     double const ratio = (double)cfinfo.fileSize / cfinfo.frameSummary.lz4FrameInfo.contentSize * 100;
1668                     DISPLAYOUT("%9.2f%%  %s \n", ratio, cfinfo.fileName);
1669                 } else {
1670                     DISPLAYOUT("%9s   %s\n",
1671                             "-",
1672                             cfinfo.fileName);
1673         }   }   }  /* if (g_displayLevel < 3) */
1674     }  /* for (; idx < ifnIdx; idx++) */
1675 
1676     return result;
1677 }
1678