Lines Matching +full:5 +full:- +full:byte
5 * This source code is licensed under both the BSD-style license (found in the
8 * You may select, at your option, one of the above-listed licenses.
11 /*-*************************************
22 /*-*************************************
26 * Stores Literals Block Type for a super-block in hType, and
32 BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE];
37 * Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and
45 BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE];
57 * Builds entropy for the super-block literals.
68 BYTE* const wkspStart = (BYTE*)workspace; in ZSTD_buildSuperBlockEntropy_literal()
69 BYTE* const wkspEnd = wkspStart + wkspSize; in ZSTD_buildSuperBlockEntropy_literal()
70 BYTE* const countWkspStart = wkspStart; in ZSTD_buildSuperBlockEntropy_literal()
73 BYTE* const nodeWksp = countWkspStart + countWkspSize; in ZSTD_buildSuperBlockEntropy_literal()
74 const size_t nodeWkspSize = wkspEnd-nodeWksp; in ZSTD_buildSuperBlockEntropy_literal()
77 HUF_repeat repeat = prevHuf->repeatMode; in ZSTD_buildSuperBlockEntropy_literal()
79 DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_literal (srcSize=%zu)", srcSize); in ZSTD_buildSuperBlockEntropy_literal()
85 DEBUGLOG(5, "set_basic - disabled"); in ZSTD_buildSuperBlockEntropy_literal()
86 hufMetadata->hType = set_basic; in ZSTD_buildSuperBlockEntropy_literal()
92 …{ size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SI… in ZSTD_buildSuperBlockEntropy_literal()
94 DEBUGLOG(5, "set_basic - too small"); in ZSTD_buildSuperBlockEntropy_literal()
95 hufMetadata->hType = set_basic; in ZSTD_buildSuperBlockEntropy_literal()
101 …{ size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)src, srcSize,… in ZSTD_buildSuperBlockEntropy_literal()
104 DEBUGLOG(5, "set_rle"); in ZSTD_buildSuperBlockEntropy_literal()
105 hufMetadata->hType = set_rle; in ZSTD_buildSuperBlockEntropy_literal()
109 DEBUGLOG(5, "set_basic - no gain"); in ZSTD_buildSuperBlockEntropy_literal()
110 hufMetadata->hType = set_basic; in ZSTD_buildSuperBlockEntropy_literal()
116 …if (repeat == HUF_repeat_check && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp,… in ZSTD_buildSuperBlockEntropy_literal()
121 ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable)); in ZSTD_buildSuperBlockEntropy_literal()
123 { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp, in ZSTD_buildSuperBlockEntropy_literal()
130 (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue); in ZSTD_buildSuperBlockEntropy_literal()
132 hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer), in ZSTD_buildSuperBlockEntropy_literal()
133 (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog, in ZSTD_buildSuperBlockEntropy_literal()
138 (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue); in ZSTD_buildSuperBlockEntropy_literal()
140 DEBUGLOG(5, "set_repeat - smaller"); in ZSTD_buildSuperBlockEntropy_literal()
142 hufMetadata->hType = set_repeat; in ZSTD_buildSuperBlockEntropy_literal()
147 DEBUGLOG(5, "set_basic - no gains"); in ZSTD_buildSuperBlockEntropy_literal()
149 hufMetadata->hType = set_basic; in ZSTD_buildSuperBlockEntropy_literal()
152 DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize); in ZSTD_buildSuperBlockEntropy_literal()
153 hufMetadata->hType = set_compressed; in ZSTD_buildSuperBlockEntropy_literal()
154 nextHuf->repeatMode = HUF_repeat_check; in ZSTD_buildSuperBlockEntropy_literal()
161 * Builds entropy for the super-block sequences.
171 BYTE* const wkspStart = (BYTE*)workspace; in ZSTD_buildSuperBlockEntropy_sequences()
172 BYTE* const wkspEnd = wkspStart + wkspSize; in ZSTD_buildSuperBlockEntropy_sequences()
173 BYTE* const countWkspStart = wkspStart; in ZSTD_buildSuperBlockEntropy_sequences()
176 BYTE* const cTableWksp = countWkspStart + countWkspSize; in ZSTD_buildSuperBlockEntropy_sequences()
177 const size_t cTableWkspSize = wkspEnd-cTableWksp; in ZSTD_buildSuperBlockEntropy_sequences()
178 ZSTD_strategy const strategy = cctxParams->cParams.strategy; in ZSTD_buildSuperBlockEntropy_sequences()
179 FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable; in ZSTD_buildSuperBlockEntropy_sequences()
180 FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable; in ZSTD_buildSuperBlockEntropy_sequences()
181 FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable; in ZSTD_buildSuperBlockEntropy_sequences()
182 const BYTE* const ofCodeTable = seqStorePtr->ofCode; in ZSTD_buildSuperBlockEntropy_sequences()
183 const BYTE* const llCodeTable = seqStorePtr->llCode; in ZSTD_buildSuperBlockEntropy_sequences()
184 const BYTE* const mlCodeTable = seqStorePtr->mlCode; in ZSTD_buildSuperBlockEntropy_sequences()
185 size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; in ZSTD_buildSuperBlockEntropy_sequences()
186 BYTE* const ostart = fseMetadata->fseTablesBuffer; in ZSTD_buildSuperBlockEntropy_sequences()
187 BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer); in ZSTD_buildSuperBlockEntropy_sequences()
188 BYTE* op = ostart; in ZSTD_buildSuperBlockEntropy_sequences()
191 DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_sequences (nbSeq=%zu)", nbSeq); in ZSTD_buildSuperBlockEntropy_sequences()
194 fseMetadata->lastCountSize = 0; in ZSTD_buildSuperBlockEntropy_sequences()
201 DEBUGLOG(5, "Building LL table"); in ZSTD_buildSuperBlockEntropy_sequences()
202 nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode; in ZSTD_buildSuperBlockEntropy_sequences()
203 LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode, in ZSTD_buildSuperBlockEntropy_sequences()
205 LLFSELog, prevEntropy->litlengthCTable, in ZSTD_buildSuperBlockEntropy_sequences()
209 …assert(!(LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We … in ZSTD_buildSuperBlockEntropy_sequences()
210 …{ size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEn… in ZSTD_buildSuperBlockEntropy_sequences()
212 … prevEntropy->litlengthCTable, sizeof(prevEntropy->litlengthCTable), in ZSTD_buildSuperBlockEntropy_sequences()
216 fseMetadata->lastCountSize = countSize; in ZSTD_buildSuperBlockEntropy_sequences()
218 fseMetadata->llType = (symbolEncodingType_e) LLtype; in ZSTD_buildSuperBlockEntropy_sequences()
226 DEBUGLOG(5, "Building OF table"); in ZSTD_buildSuperBlockEntropy_sequences()
227 nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode; in ZSTD_buildSuperBlockEntropy_sequences()
228 Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode, in ZSTD_buildSuperBlockEntropy_sequences()
230 OffFSELog, prevEntropy->offcodeCTable, in ZSTD_buildSuperBlockEntropy_sequences()
233 …assert(!(Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We d… in ZSTD_buildSuperBlockEntropy_sequences()
234 …{ size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbol… in ZSTD_buildSuperBlockEntropy_sequences()
236 … prevEntropy->offcodeCTable, sizeof(prevEntropy->offcodeCTable), in ZSTD_buildSuperBlockEntropy_sequences()
240 fseMetadata->lastCountSize = countSize; in ZSTD_buildSuperBlockEntropy_sequences()
242 fseMetadata->ofType = (symbolEncodingType_e) Offtype; in ZSTD_buildSuperBlockEntropy_sequences()
248 DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); in ZSTD_buildSuperBlockEntropy_sequences()
249 nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode; in ZSTD_buildSuperBlockEntropy_sequences()
250 MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode, in ZSTD_buildSuperBlockEntropy_sequences()
252 MLFSELog, prevEntropy->matchlengthCTable, in ZSTD_buildSuperBlockEntropy_sequences()
255 …assert(!(MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* W… in ZSTD_buildSuperBlockEntropy_sequences()
256 …{ size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbol… in ZSTD_buildSuperBlockEntropy_sequences()
258 … prevEntropy->matchlengthCTable, sizeof(prevEntropy->matchlengthCTable), in ZSTD_buildSuperBlockEntropy_sequences()
262 fseMetadata->lastCountSize = countSize; in ZSTD_buildSuperBlockEntropy_sequences()
264 fseMetadata->mlType = (symbolEncodingType_e) MLtype; in ZSTD_buildSuperBlockEntropy_sequences()
266 assert((size_t) (op-ostart) <= sizeof(fseMetadata->fseTablesBuffer)); in ZSTD_buildSuperBlockEntropy_sequences()
267 return op-ostart; in ZSTD_buildSuperBlockEntropy_sequences()
272 * Builds entropy for the super-block.
282 size_t const litSize = seqStorePtr->lit - seqStorePtr->litStart; in ZSTD_buildSuperBlockEntropy()
283 DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy"); in ZSTD_buildSuperBlockEntropy()
284 entropyMetadata->hufMetadata.hufDesSize = in ZSTD_buildSuperBlockEntropy()
285 ZSTD_buildSuperBlockEntropy_literal(seqStorePtr->litStart, litSize, in ZSTD_buildSuperBlockEntropy()
286 &prevEntropy->huf, &nextEntropy->huf, in ZSTD_buildSuperBlockEntropy()
287 &entropyMetadata->hufMetadata, in ZSTD_buildSuperBlockEntropy()
290 …FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildSuperBlockEntropy_literal fai… in ZSTD_buildSuperBlockEntropy()
291 entropyMetadata->fseMetadata.fseTablesSize = in ZSTD_buildSuperBlockEntropy()
293 &prevEntropy->fse, &nextEntropy->fse, in ZSTD_buildSuperBlockEntropy()
295 &entropyMetadata->fseMetadata, in ZSTD_buildSuperBlockEntropy()
297 …FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildSuperBlockEntropy_sequence… in ZSTD_buildSuperBlockEntropy()
302 * Compresses literals section for a sub-block.
311 * hufMetadata->hType has literals block type info.
312 * If it is set_basic, all sub-blocks literals section will be Raw_Literals_Block.
313 * If it is set_rle, all sub-blocks literals section will be RLE_Literals_Block.
314 …* If it is set_compressed, first sub-block's literals section will be Compressed_Literals_Blo…
315 * If it is set_compressed, first sub-block's literals section will be Treeless_Literals_Block
316 * and the following sub-blocks' literals sections will be Treeless_Literals_Block.
317 * @return : compressed size of literals section of a sub-block
322 const BYTE* literals, size_t litSize, in ZSTD_compressSubBlock_literal()
327 size_t const lhSize = 3 + (litSize >= (1 KB - header)) + (litSize >= (16 KB - header)); in ZSTD_compressSubBlock_literal()
328 BYTE* const ostart = (BYTE*)dst; in ZSTD_compressSubBlock_literal()
329 BYTE* const oend = ostart + dstSize; in ZSTD_compressSubBlock_literal()
330 BYTE* op = ostart + lhSize; in ZSTD_compressSubBlock_literal()
332 symbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat; in ZSTD_compressSubBlock_literal()
337 …DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, l… in ZSTD_compressSubBlock_literal()
340 if (litSize == 0 || hufMetadata->hType == set_basic) { in ZSTD_compressSubBlock_literal()
341 DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal"); in ZSTD_compressSubBlock_literal()
343 } else if (hufMetadata->hType == set_rle) { in ZSTD_compressSubBlock_literal()
344 DEBUGLOG(5, "ZSTD_compressSubBlock_literal using rle literal"); in ZSTD_compressSubBlock_literal()
349 assert(hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat); in ZSTD_compressSubBlock_literal()
351 if (writeEntropy && hufMetadata->hType == set_compressed) { in ZSTD_compressSubBlock_literal()
352 ZSTD_memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize); in ZSTD_compressSubBlock_literal()
353 op += hufMetadata->hufDesSize; in ZSTD_compressSubBlock_literal()
354 cLitSize += hufMetadata->hufDesSize; in ZSTD_compressSubBlock_literal()
355 DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize); in ZSTD_compressSubBlock_literal()
359 …{ const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize,… in ZSTD_compressSubBlock_literal()
360 … : HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable); in ZSTD_compressSubBlock_literal()
364 DEBUGLOG(5, "Failed to write entropy tables %s", ZSTD_getErrorName(cSize)); in ZSTD_compressSubBlock_literal()
369 DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal because uncompressible"); in ZSTD_compressSubBlock_literal()
375 DEBUGLOG(5, "Literals expanded beyond allowed header size"); in ZSTD_compressSubBlock_literal()
378 DEBUGLOG(5, "ZSTD_compressSubBlock_literal (cSize=%zu)", cSize); in ZSTD_compressSubBlock_literal()
384 case 3: /* 2 - 2 - 10 - 10 */ in ZSTD_compressSubBlock_literal()
389 case 4: /* 2 - 2 - 14 - 14 */ in ZSTD_compressSubBlock_literal()
394 case 5: /* 2 - 2 - 18 - 18 */ in ZSTD_compressSubBlock_literal()
397 ostart[4] = (BYTE)(cLitSize >> 10); in ZSTD_compressSubBlock_literal()
400 default: /* not possible : lhSize is {3,4,5} */ in ZSTD_compressSubBlock_literal()
404 DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)litSize, (U32)(op-ostart)); in ZSTD_compressSubBlock_literal()
405 return op-ostart; in ZSTD_compressSubBlock_literal()
416 while (send-sp > 0) { in ZSTD_seqDecompressedSize()
430 * Compresses sequences section for a sub-block.
431 * fseMetadata->llType, fseMetadata->ofType, and fseMetadata->mlType have
432 * symbol compression modes for the super-block.
435 * The following sub-blocks will always have repeat mode.
436 * @return : compressed size of sequences section of a sub-block
442 … const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, in ZSTD_compressSubBlock_sequences()
447 const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; in ZSTD_compressSubBlock_sequences()
448 BYTE* const ostart = (BYTE*)dst; in ZSTD_compressSubBlock_sequences()
449 BYTE* const oend = ostart + dstCapacity; in ZSTD_compressSubBlock_sequences()
450 BYTE* op = ostart; in ZSTD_compressSubBlock_sequences()
451 BYTE* seqHead; in ZSTD_compressSubBlock_sequences()
453 …DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (nbSeq=%zu, writeEntropy=%d, longOffsets=%d)", nbSeq,… in ZSTD_compressSubBlock_sequences()
457 RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, in ZSTD_compressSubBlock_sequences()
460 *op++ = (BYTE)nbSeq; in ZSTD_compressSubBlock_sequences()
462 op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; in ZSTD_compressSubBlock_sequences()
464 op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; in ZSTD_compressSubBlock_sequences()
466 return op - ostart; in ZSTD_compressSubBlock_sequences()
472 DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (seqHeadSize=%u)", (unsigned)(op-ostart)); in ZSTD_compressSubBlock_sequences()
475 const U32 LLtype = fseMetadata->llType; in ZSTD_compressSubBlock_sequences()
476 const U32 Offtype = fseMetadata->ofType; in ZSTD_compressSubBlock_sequences()
477 const U32 MLtype = fseMetadata->mlType; in ZSTD_compressSubBlock_sequences()
478 … DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (fseTablesSize=%zu)", fseMetadata->fseTablesSize); in ZSTD_compressSubBlock_sequences()
479 *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); in ZSTD_compressSubBlock_sequences()
480 ZSTD_memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize); in ZSTD_compressSubBlock_sequences()
481 op += fseMetadata->fseTablesSize; in ZSTD_compressSubBlock_sequences()
484 *seqHead = (BYTE)((repeat<<6) + (repeat<<4) + (repeat<<2)); in ZSTD_compressSubBlock_sequences()
488 op, oend - op, in ZSTD_compressSubBlock_sequences()
489 fseTables->matchlengthCTable, mlCode, in ZSTD_compressSubBlock_sequences()
490 fseTables->offcodeCTable, ofCode, in ZSTD_compressSubBlock_sequences()
491 fseTables->litlengthCTable, llCode, in ZSTD_compressSubBlock_sequences()
500 * bytes and the bitstream is only one byte. in ZSTD_compressSubBlock_sequences()
505 …if (writeEntropy && fseMetadata->lastCountSize && fseMetadata->lastCountSize + bitstreamSize < 4) { in ZSTD_compressSubBlock_sequences()
507 assert(fseMetadata->lastCountSize + bitstreamSize == 3); in ZSTD_compressSubBlock_sequences()
508 DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " in ZSTD_compressSubBlock_sequences()
513 DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (bitstreamSize=%zu)", bitstreamSize); in ZSTD_compressSubBlock_sequences()
521 * with repeat mode where sequences section body size can be 1 byte. in ZSTD_compressSubBlock_sequences()
524 if (op-seqHead < 4) { in ZSTD_compressSubBlock_sequences()
525 DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.4.0 by emitting " in ZSTD_compressSubBlock_sequences()
532 return op - ostart; in ZSTD_compressSubBlock_sequences()
536 * Compresses a single sub-block.
537 * @return : compressed size of the sub-block
542 const BYTE* literals, size_t litSize, in ZSTD_compressSubBlock()
543 const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, in ZSTD_compressSubBlock()
551 BYTE* const ostart = (BYTE*)dst; in ZSTD_compressSubBlock()
552 BYTE* const oend = ostart + dstCapacity; in ZSTD_compressSubBlock()
553 BYTE* op = ostart + ZSTD_blockHeaderSize; in ZSTD_compressSubBlock()
554 …DEBUGLOG(5, "ZSTD_compressSubBlock (litSize=%zu, nbSeq=%zu, writeLitEntropy=%d, writeSeqEntropy=%d… in ZSTD_compressSubBlock()
556 { size_t cLitSize = ZSTD_compressSubBlock_literal((const HUF_CElt*)entropy->huf.CTable, in ZSTD_compressSubBlock()
557 … &entropyMetadata->hufMetadata, literals, litSize, in ZSTD_compressSubBlock()
558 … op, oend-op, bmi2, writeLitEntropy, litEntropyWritten); in ZSTD_compressSubBlock()
563 { size_t cSeqSize = ZSTD_compressSubBlock_sequences(&entropy->fse, in ZSTD_compressSubBlock()
564 &entropyMetadata->fseMetadata, in ZSTD_compressSubBlock()
568 op, oend-op, in ZSTD_compressSubBlock()
575 { size_t cSize = (op-ostart)-ZSTD_blockHeaderSize; in ZSTD_compressSubBlock()
579 return op-ostart; in ZSTD_compressSubBlock()
582 static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t litSize, in ZSTD_estimateSubBlockSize_literal()
592 if (hufMetadata->hType == set_basic) return litSize; in ZSTD_estimateSubBlockSize_literal()
593 else if (hufMetadata->hType == set_rle) return 1; in ZSTD_estimateSubBlockSize_literal()
594 else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) { in ZSTD_estimateSubBlockSize_literal()
595 …size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize… in ZSTD_estimateSubBlockSize_literal()
597 …{ size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, … in ZSTD_estimateSubBlockSize_literal()
598 if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize; in ZSTD_estimateSubBlockSize_literal()
606 const BYTE* codeTable, unsigned maxCode, in ZSTD_estimateSubBlockSize_symbolType()
613 const BYTE* ctp = codeTable; in ZSTD_estimateSubBlockSize_symbolType()
614 const BYTE* const ctStart = ctp; in ZSTD_estimateSubBlockSize_symbolType()
615 const BYTE* const ctEnd = ctStart + nbSeq; in ZSTD_estimateSubBlockSize_symbolType()
640 static size_t ZSTD_estimateSubBlockSize_sequences(const BYTE* ofCodeTable, in ZSTD_estimateSubBlockSize_sequences()
641 const BYTE* llCodeTable, in ZSTD_estimateSubBlockSize_sequences()
642 const BYTE* mlCodeTable, in ZSTD_estimateSubBlockSize_sequences()
651 … cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, MaxOff, in ZSTD_estimateSubBlockSize_sequences()
652 nbSeq, fseTables->offcodeCTable, NULL, in ZSTD_estimateSubBlockSize_sequences()
655 … cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->llType, llCodeTable, MaxLL, in ZSTD_estimateSubBlockSize_sequences()
656 nbSeq, fseTables->litlengthCTable, LL_bits, in ZSTD_estimateSubBlockSize_sequences()
659 … cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, MaxML, in ZSTD_estimateSubBlockSize_sequences()
660 nbSeq, fseTables->matchlengthCTable, ML_bits, in ZSTD_estimateSubBlockSize_sequences()
663 if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize; in ZSTD_estimateSubBlockSize_sequences()
667 static size_t ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize, in ZSTD_estimateSubBlockSize()
668 const BYTE* ofCodeTable, in ZSTD_estimateSubBlockSize()
669 const BYTE* llCodeTable, in ZSTD_estimateSubBlockSize()
670 const BYTE* mlCodeTable, in ZSTD_estimateSubBlockSize()
678 … &entropy->huf, &entropyMetadata->hufMetadata, in ZSTD_estimateSubBlockSize()
681 … nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, in ZSTD_estimateSubBlockSize()
688 if (fseMetadata->llType == set_compressed || fseMetadata->llType == set_rle) in ZSTD_needSequenceEntropyTables()
690 if (fseMetadata->mlType == set_compressed || fseMetadata->mlType == set_rle) in ZSTD_needSequenceEntropyTables()
692 if (fseMetadata->ofType == set_compressed || fseMetadata->ofType == set_rle) in ZSTD_needSequenceEntropyTables()
698 * Breaks super-block into multiple sub-blocks and compresses them.
701 * All sub-blocks are compressed blocks (no raw or rle blocks).
714 const seqDef* const sstart = seqStorePtr->sequencesStart; in ZSTD_compressSubBlock_multi()
715 const seqDef* const send = seqStorePtr->sequences; in ZSTD_compressSubBlock_multi()
717 const BYTE* const lstart = seqStorePtr->litStart; in ZSTD_compressSubBlock_multi()
718 const BYTE* const lend = seqStorePtr->lit; in ZSTD_compressSubBlock_multi()
719 const BYTE* lp = lstart; in ZSTD_compressSubBlock_multi()
720 BYTE const* ip = (BYTE const*)src; in ZSTD_compressSubBlock_multi()
721 BYTE const* const iend = ip + srcSize; in ZSTD_compressSubBlock_multi()
722 BYTE* const ostart = (BYTE*)dst; in ZSTD_compressSubBlock_multi()
723 BYTE* const oend = ostart + dstCapacity; in ZSTD_compressSubBlock_multi()
724 BYTE* op = ostart; in ZSTD_compressSubBlock_multi()
725 const BYTE* llCodePtr = seqStorePtr->llCode; in ZSTD_compressSubBlock_multi()
726 const BYTE* mlCodePtr = seqStorePtr->mlCode; in ZSTD_compressSubBlock_multi()
727 const BYTE* ofCodePtr = seqStorePtr->ofCode; in ZSTD_compressSubBlock_multi()
728 size_t targetCBlockSize = cctxParams->targetCBlockSize; in ZSTD_compressSubBlock_multi()
730 int writeLitEntropy = entropyMetadata->hufMetadata.hType == set_compressed; in ZSTD_compressSubBlock_multi()
734 DEBUGLOG(5, "ZSTD_compressSubBlock_multi (litSize=%u, nbSeq=%u)", in ZSTD_compressSubBlock_multi()
735 (unsigned)(lend-lp), (unsigned)(send-sstart)); in ZSTD_compressSubBlock_multi()
745 lastSequence = sequence == send - 1; in ZSTD_compressSubBlock_multi()
751 assert(litSize <= (size_t)(lend - lp)); in ZSTD_compressSubBlock_multi()
752 litSize = (size_t)(lend - lp); in ZSTD_compressSubBlock_multi()
760 &nextCBlock->entropy, entropyMetadata, in ZSTD_compressSubBlock_multi()
766 const size_t cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata, in ZSTD_compressSubBlock_multi()
771 op, oend-op, in ZSTD_compressSubBlock_multi()
777 DEBUGLOG(5, "Committed the sub-block"); in ZSTD_compressSubBlock_multi()
799 DEBUGLOG(5, "ZSTD_compressSubBlock_multi has literal entropy tables unwritten"); in ZSTD_compressSubBlock_multi()
800 … ZSTD_memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf)); in ZSTD_compressSubBlock_multi()
802 if (writeSeqEntropy && ZSTD_needSequenceEntropyTables(&entropyMetadata->fseMetadata)) { in ZSTD_compressSubBlock_multi()
806 DEBUGLOG(5, "ZSTD_compressSubBlock_multi has sequence entropy tables unwritten"); in ZSTD_compressSubBlock_multi()
810 size_t const cSize = ZSTD_noCompressBlock(op, oend - op, ip, iend - ip, lastBlock); in ZSTD_compressSubBlock_multi()
811 …DEBUGLOG(5, "ZSTD_compressSubBlock_multi last sub-block uncompressed, %zu bytes", (size_t)(iend - … in ZSTD_compressSubBlock_multi()
819 ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep)); in ZSTD_compressSubBlock_multi()
821 …rep = ZSTD_updateRep(rep.rep, seq->offset - 1, ZSTD_getSequenceLength(seqStorePtr, seq).litLength … in ZSTD_compressSubBlock_multi()
823 ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep)); in ZSTD_compressSubBlock_multi()
826 DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed"); in ZSTD_compressSubBlock_multi()
827 return op-ostart; in ZSTD_compressSubBlock_multi()
836 FORWARD_IF_ERROR(ZSTD_buildSuperBlockEntropy(&zc->seqStore, in ZSTD_compressSuperBlock()
837 &zc->blockState.prevCBlock->entropy, in ZSTD_compressSuperBlock()
838 &zc->blockState.nextCBlock->entropy, in ZSTD_compressSuperBlock()
839 &zc->appliedParams, in ZSTD_compressSuperBlock()
841 … zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), ""); in ZSTD_compressSuperBlock()
843 return ZSTD_compressSubBlock_multi(&zc->seqStore, in ZSTD_compressSuperBlock()
844 zc->blockState.prevCBlock, in ZSTD_compressSuperBlock()
845 zc->blockState.nextCBlock, in ZSTD_compressSuperBlock()
847 &zc->appliedParams, in ZSTD_compressSuperBlock()
850 zc->bmi2, lastBlock, in ZSTD_compressSuperBlock()
851 zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */); in ZSTD_compressSuperBlock()