1 /* LZ4frame API example : compress a file
2  * Modified from an example code by Zbigniew Jędrzejewski-Szmek
3  *
4  * This example streams an input file into an output file
5  * using a bounded memory budget.
6  * Input is read in chunks of IN_CHUNK_SIZE */
7 
8 #include <stdio.h>
9 #include <stdlib.h>
10 #include <string.h>
11 #include <errno.h>
12 #include <assert.h>
13 
14 #include <lz4frame.h>
15 
16 
17 #define IN_CHUNK_SIZE  (16*1024)
18 
19 static const LZ4F_preferences_t kPrefs = {
20     { LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame,
21       0 /* unknown content size */, 0 /* no dictID */ , LZ4F_noBlockChecksum },
22     0,   /* compression level; 0 == default */
23     0,   /* autoflush */
24     0,   /* favor decompression speed */
25     { 0, 0, 0 },  /* reserved, must be set to 0 */
26 };
27 
28 
29 /* safe_fwrite() :
30  * performs fwrite(), ensure operation success, or immediately exit() */
safe_fwrite(void * buf,size_t eltSize,size_t nbElt,FILE * f)31 static void safe_fwrite(void* buf, size_t eltSize, size_t nbElt, FILE* f)
32 {
33     size_t const writtenSize = fwrite(buf, eltSize, nbElt, f);
34     size_t const expectedSize = eltSize * nbElt;
35     if (nbElt>0) assert(expectedSize / nbElt == eltSize);  /* check overflow */
36     if (writtenSize < expectedSize) {
37         if (ferror(f))  /* note : ferror() must follow fwrite */
38             fprintf(stderr, "Write failed \n");
39         else
40             fprintf(stderr, "Write too short \n");
41         exit(1);
42     }
43 }
44 
45 
46 /* ================================================= */
47 /*     Streaming Compression example               */
48 /* ================================================= */
49 
50 typedef struct {
51     int error;
52     unsigned long long size_in;
53     unsigned long long size_out;
54 } compressResult_t;
55 
56 static compressResult_t
compress_file_internal(FILE * f_in,FILE * f_out,LZ4F_compressionContext_t ctx,void * inBuff,size_t inChunkSize,void * outBuff,size_t outCapacity)57 compress_file_internal(FILE* f_in, FILE* f_out,
58                        LZ4F_compressionContext_t ctx,
59                        void* inBuff,  size_t inChunkSize,
60                        void* outBuff, size_t outCapacity)
61 {
62     compressResult_t result = { 1, 0, 0 };  /* result for an error */
63     unsigned long long count_in = 0, count_out;
64 
65     assert(f_in != NULL); assert(f_out != NULL);
66     assert(ctx != NULL);
67     assert(outCapacity >= LZ4F_HEADER_SIZE_MAX);
68     assert(outCapacity >= LZ4F_compressBound(inChunkSize, &kPrefs));
69 
70     /* write frame header */
71     {   size_t const headerSize = LZ4F_compressBegin(ctx, outBuff, outCapacity, &kPrefs);
72         if (LZ4F_isError(headerSize)) {
73             printf("Failed to start compression: error %u \n", (unsigned)headerSize);
74             return result;
75         }
76         count_out = headerSize;
77         printf("Buffer size is %u bytes, header size %u bytes \n",
78                 (unsigned)outCapacity, (unsigned)headerSize);
79         safe_fwrite(outBuff, 1, headerSize, f_out);
80     }
81 
82     /* stream file */
83     for (;;) {
84         size_t const readSize = fread(inBuff, 1, IN_CHUNK_SIZE, f_in);
85         if (readSize == 0) break; /* nothing left to read from input file */
86         count_in += readSize;
87 
88         size_t const compressedSize = LZ4F_compressUpdate(ctx,
89                                                 outBuff, outCapacity,
90                                                 inBuff, readSize,
91                                                 NULL);
92         if (LZ4F_isError(compressedSize)) {
93             printf("Compression failed: error %u \n", (unsigned)compressedSize);
94             return result;
95         }
96 
97         printf("Writing %u bytes\n", (unsigned)compressedSize);
98         safe_fwrite(outBuff, 1, compressedSize, f_out);
99         count_out += compressedSize;
100     }
101 
102     /* flush whatever remains within internal buffers */
103     {   size_t const compressedSize = LZ4F_compressEnd(ctx,
104                                                 outBuff, outCapacity,
105                                                 NULL);
106         if (LZ4F_isError(compressedSize)) {
107             printf("Failed to end compression: error %u \n", (unsigned)compressedSize);
108             return result;
109         }
110 
111         printf("Writing %u bytes \n", (unsigned)compressedSize);
112         safe_fwrite(outBuff, 1, compressedSize, f_out);
113         count_out += compressedSize;
114     }
115 
116     result.size_in = count_in;
117     result.size_out = count_out;
118     result.error = 0;
119     return result;
120 }
121 
122 static compressResult_t
compress_file(FILE * f_in,FILE * f_out)123 compress_file(FILE* f_in, FILE* f_out)
124 {
125     assert(f_in != NULL);
126     assert(f_out != NULL);
127 
128     /* ressource allocation */
129     LZ4F_compressionContext_t ctx;
130     size_t const ctxCreation = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
131     void* const src = malloc(IN_CHUNK_SIZE);
132     size_t const outbufCapacity = LZ4F_compressBound(IN_CHUNK_SIZE, &kPrefs);   /* large enough for any input <= IN_CHUNK_SIZE */
133     void* const outbuff = malloc(outbufCapacity);
134 
135     compressResult_t result = { 1, 0, 0 };  /* == error (default) */
136     if (!LZ4F_isError(ctxCreation) && src && outbuff) {
137         result = compress_file_internal(f_in, f_out,
138                                         ctx,
139                                         src, IN_CHUNK_SIZE,
140                                         outbuff, outbufCapacity);
141     } else {
142         printf("error : ressource allocation failed \n");
143     }
144 
145     LZ4F_freeCompressionContext(ctx);   /* supports free on NULL */
146     free(src);
147     free(outbuff);
148     return result;
149 }
150 
151 
152 /* ================================================= */
153 /*     Streaming decompression example               */
154 /* ================================================= */
155 
get_block_size(const LZ4F_frameInfo_t * info)156 static size_t get_block_size(const LZ4F_frameInfo_t* info) {
157     switch (info->blockSizeID) {
158         case LZ4F_default:
159         case LZ4F_max64KB:  return 1 << 16;
160         case LZ4F_max256KB: return 1 << 18;
161         case LZ4F_max1MB:   return 1 << 20;
162         case LZ4F_max4MB:   return 1 << 22;
163         default:
164             printf("Impossible with expected frame specification (<=v1.6.1)\n");
165             exit(1);
166     }
167 }
168 
169 /* @return : 1==error, 0==success */
170 static int
decompress_file_internal(FILE * f_in,FILE * f_out,LZ4F_dctx * dctx,void * src,size_t srcCapacity,size_t filled,size_t alreadyConsumed,void * dst,size_t dstCapacity)171 decompress_file_internal(FILE* f_in, FILE* f_out,
172                          LZ4F_dctx* dctx,
173                          void* src, size_t srcCapacity, size_t filled, size_t alreadyConsumed,
174                          void* dst, size_t dstCapacity)
175 {
176     int firstChunk = 1;
177     size_t ret = 1;
178 
179     assert(f_in != NULL); assert(f_out != NULL);
180     assert(dctx != NULL);
181     assert(src != NULL); assert(srcCapacity > 0); assert(filled <= srcCapacity); assert(alreadyConsumed <= filled);
182     assert(dst != NULL); assert(dstCapacity > 0);
183 
184     /* Decompression */
185     while (ret != 0) {
186         /* Load more input */
187         size_t readSize = firstChunk ? filled : fread(src, 1, srcCapacity, f_in); firstChunk=0;
188         const void* srcPtr = (const char*)src + alreadyConsumed; alreadyConsumed=0;
189         const void* const srcEnd = (const char*)srcPtr + readSize;
190         if (readSize == 0 || ferror(f_in)) {
191             printf("Decompress: not enough input or error reading file\n");
192             return 1;
193         }
194 
195         /* Decompress:
196          * Continue while there is more input to read (srcPtr != srcEnd)
197          * and the frame isn't over (ret != 0)
198          */
199         while (srcPtr < srcEnd && ret != 0) {
200             /* Any data within dst has been flushed at this stage */
201             size_t dstSize = dstCapacity;
202             size_t srcSize = (const char*)srcEnd - (const char*)srcPtr;
203             ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL);
204             if (LZ4F_isError(ret)) {
205                 printf("Decompression error: %s\n", LZ4F_getErrorName(ret));
206                 return 1;
207             }
208             /* Flush output */
209             if (dstSize != 0) safe_fwrite(dst, 1, dstSize, f_out);
210             /* Update input */
211             srcPtr = (const char*)srcPtr + srcSize;
212         }
213 
214         assert(srcPtr <= srcEnd);
215 
216         /* Ensure all input data has been consumed.
217          * It is valid to have multiple frames in the same file,
218          * but this example only supports one frame.
219          */
220         if (srcPtr < srcEnd) {
221             printf("Decompress: Trailing data left in file after frame\n");
222             return 1;
223         }
224     }
225 
226     /* Check that there isn't trailing data in the file after the frame.
227      * It is valid to have multiple frames in the same file,
228      * but this example only supports one frame.
229      */
230     {   size_t const readSize = fread(src, 1, 1, f_in);
231         if (readSize != 0 || !feof(f_in)) {
232             printf("Decompress: Trailing data left in file after frame\n");
233             return 1;
234     }   }
235 
236     return 0;
237 }
238 
239 
240 /* @return : 1==error, 0==completed */
241 static int
decompress_file_allocDst(FILE * f_in,FILE * f_out,LZ4F_dctx * dctx,void * src,size_t srcCapacity)242 decompress_file_allocDst(FILE* f_in, FILE* f_out,
243                         LZ4F_dctx* dctx,
244                         void* src, size_t srcCapacity)
245 {
246     assert(f_in != NULL); assert(f_out != NULL);
247     assert(dctx != NULL);
248     assert(src != NULL);
249     assert(srcCapacity >= LZ4F_HEADER_SIZE_MAX);  /* ensure LZ4F_getFrameInfo() can read enough data */
250 
251     /* Read Frame header */
252     size_t const readSize = fread(src, 1, srcCapacity, f_in);
253     if (readSize == 0 || ferror(f_in)) {
254         printf("Decompress: not enough input or error reading file\n");
255         return 1;
256     }
257 
258     LZ4F_frameInfo_t info;
259     size_t consumedSize = readSize;
260     {   size_t const fires = LZ4F_getFrameInfo(dctx, &info, src, &consumedSize);
261         if (LZ4F_isError(fires)) {
262             printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(fires));
263             return 1;
264     }   }
265 
266     /* Allocating enough space for an entire block isn't necessary for
267      * correctness, but it allows some memcpy's to be elided.
268      */
269     size_t const dstCapacity = get_block_size(&info);
270     void* const dst = malloc(dstCapacity);
271     if (!dst) { perror("decompress_file(dst)"); return 1; }
272 
273     int const decompressionResult = decompress_file_internal(
274                         f_in, f_out,
275                         dctx,
276                         src, srcCapacity, readSize-consumedSize, consumedSize,
277                         dst, dstCapacity);
278 
279     free(dst);
280     return decompressionResult;
281 }
282 
283 
284 /* @result : 1==error, 0==success */
decompress_file(FILE * f_in,FILE * f_out)285 static int decompress_file(FILE* f_in, FILE* f_out)
286 {
287     assert(f_in != NULL); assert(f_out != NULL);
288 
289     /* Ressource allocation */
290     void* const src = malloc(IN_CHUNK_SIZE);
291     if (!src) { perror("decompress_file(src)"); return 1; }
292 
293     LZ4F_dctx* dctx;
294     {   size_t const dctxStatus = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
295         if (LZ4F_isError(dctxStatus)) {
296             printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(dctxStatus));
297     }   }
298 
299     int const result = !dctx ? 1 /* error */ :
300                        decompress_file_allocDst(f_in, f_out, dctx, src, IN_CHUNK_SIZE);
301 
302     free(src);
303     LZ4F_freeDecompressionContext(dctx);   /* note : free works on NULL */
304     return result;
305 }
306 
307 
compareFiles(FILE * fp0,FILE * fp1)308 int compareFiles(FILE* fp0, FILE* fp1)
309 {
310     int result = 0;
311 
312     while (result==0) {
313         char b0[1024];
314         char b1[1024];
315         size_t const r0 = fread(b0, 1, sizeof(b0), fp0);
316         size_t const r1 = fread(b1, 1, sizeof(b1), fp1);
317 
318         result = (r0 != r1);
319         if (!r0 || !r1) break;
320         if (!result) result = memcmp(b0, b1, r0);
321     }
322 
323     return result;
324 }
325 
326 
main(int argc,const char ** argv)327 int main(int argc, const char **argv) {
328     char inpFilename[256] = { 0 };
329     char lz4Filename[256] = { 0 };
330     char decFilename[256] = { 0 };
331 
332     if (argc < 2) {
333         printf("Please specify input filename\n");
334         return 0;
335     }
336 
337     snprintf(inpFilename, 256, "%s", argv[1]);
338     snprintf(lz4Filename, 256, "%s.lz4", argv[1]);
339     snprintf(decFilename, 256, "%s.lz4.dec", argv[1]);
340 
341     printf("inp = [%s]\n", inpFilename);
342     printf("lz4 = [%s]\n", lz4Filename);
343     printf("dec = [%s]\n", decFilename);
344 
345     /* compress */
346     {   FILE* const inpFp = fopen(inpFilename, "rb");
347         FILE* const outFp = fopen(lz4Filename, "wb");
348 
349         printf("compress : %s -> %s\n", inpFilename, lz4Filename);
350         compressResult_t const ret = compress_file(inpFp, outFp);
351 
352         fclose(outFp);
353         fclose(inpFp);
354 
355         if (ret.error) {
356             printf("compress : failed with code %i\n", ret.error);
357             return ret.error;
358         }
359         printf("%s: %zu → %zu bytes, %.1f%%\n",
360             inpFilename,
361             (size_t)ret.size_in, (size_t)ret.size_out,  /* might overflow is size_t is 32 bits and size_{in,out} > 4 GB */
362             (double)ret.size_out / ret.size_in * 100);
363         printf("compress : done\n");
364     }
365 
366     /* decompress */
367     {   FILE* const inpFp = fopen(lz4Filename, "rb");
368         FILE* const outFp = fopen(decFilename, "wb");
369 
370         printf("decompress : %s -> %s\n", lz4Filename, decFilename);
371         int const ret = decompress_file(inpFp, outFp);
372 
373         fclose(outFp);
374         fclose(inpFp);
375 
376         if (ret) {
377             printf("decompress : failed with code %i\n", ret);
378             return ret;
379         }
380         printf("decompress : done\n");
381     }
382 
383     /* verify */
384     {   FILE* const inpFp = fopen(inpFilename, "rb");
385         FILE* const decFp = fopen(decFilename, "rb");
386 
387         printf("verify : %s <-> %s\n", inpFilename, decFilename);
388         int const cmp = compareFiles(inpFp, decFp);
389 
390         fclose(decFp);
391         fclose(inpFp);
392 
393         if (cmp) {
394             printf("corruption detected : decompressed file differs from original\n");
395             return cmp;
396         }
397         printf("verify : OK\n");
398     }
399 
400     return 0;
401 }
402