Skip to content

opt: tabulate the optimal parser's per-code price components #4732

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
matevz-kovacic wants to merge 1 commit into
base: dev
Choose a base branch
from
Open

opt: tabulate the optimal parser's per-code price components #4732

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions lib/compress/zstd_compress_internal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,24 @@ typedef struct {
ZSTD_OptPrice_e priceType; /* prices can be determined dynamically, or follow a pre-defined cost structure */
const ZSTD_entropyCTables_t* symbolCosts; /* pre-calculated dictionary statistics */
ZSTD_ParamSwitch_e literalCompressionMode;

/* Derived price tables: the per-code half of a price that depends on
* that code's frequency alone. The matching `*SumBasePrice` above is
* added at the point of use, which is what keeps these tables cheap to
* maintain: a frequency update touches exactly one entry per table, so
* ZSTD_updateStats() refreshes those entries in place and no full
* rebuild is needed when the running totals move. A full build is only
* required when ZSTD_rescaleFreqs() replaces all the statistics at
* once, i.e. once per block.
*
* Entries are U32 and deliberately allowed to wrap: each is a
* difference that is only meaningful once its `*SumBasePrice` is added
* back, and U32 addition recovers the original value exactly. These
* tables therefore reproduce the arithmetic they replace bit for bit,
* leaving the parse unchanged. */
U32 litLengthPriceNoSum[MaxLL+1]; /* + litLengthSumBasePrice = litLength symbol price */
U32 matchLengthPriceNoSum[MaxML+1]; /* + matchLengthSumBasePrice = matchLength half of a match price */
U32 offCodePriceNoSum[MaxOff+1]; /* + offCodeSumBasePrice = offset half of a match price */
} optState_t;

typedef struct {
Expand Down
188 changes: 153 additions & 35 deletions lib/compress/zstd_opt.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,61 @@ static int ZSTD_compressedLiterals(optState_t const* const optPtr)
return optPtr->literalCompressionMode != ZSTD_ps_disable;
}

/* The per-code price halves.
*
* Within a stretch group the statistics are frozen, so a symbol's price is
* a pure function of its *code*: 36 litLength codes, 53 matchLength codes,
* 32 offset codes. The parser nevertheless re-derived each one from the raw
* frequency arrays at every evaluation -- an array load plus a fixed-point
* log approximation, twice per candidate match length -- inside a loop that
* runs once per (match, length) pair. These three helpers compute the same
* value once per frequency change instead.
*
* What is stored deliberately EXCLUDES the `*SumBasePrice` term, which is
* added back at the point of use. That is what makes maintenance cheap: a
* statistics update changes one frequency (one entry) and the running total
* (one scalar), so nothing has to be rebuilt wholesale. The two terms the
* match price used to add after the per-code terms -- BITCOST_MULTIPLIER/5,
* and the long-offset handicap -- are folded in here; U32 addition is
* associative, so neither folding them in nor deferring the sum term can
* change a price by one unit. */

static U32 ZSTD_litLengthPriceNoSum(const optState_t* optPtr, U32 llCode, int optLevel)
{
return (LL_bits[llCode] * BITCOST_MULTIPLIER)
- WEIGHT(optPtr->litLengthFreq[llCode], optLevel);
}

static U32 ZSTD_matchLengthPriceNoSum(const optState_t* optPtr, U32 mlCode, int optLevel)
{
return (ML_bits[mlCode] * BITCOST_MULTIPLIER)
- WEIGHT(optPtr->matchLengthFreq[mlCode], optLevel)
+ BITCOST_MULTIPLIER / 5; /* heuristic : make matches a bit more costly to favor less sequences -> faster decompression speed */
}

static U32 ZSTD_offCodePriceNoSum(const optState_t* optPtr, U32 offCode, int optLevel)
{
U32 price = (offCode * BITCOST_MULTIPLIER)
- WEIGHT(optPtr->offCodeFreq[offCode], optLevel);
if ((optLevel < 2) /*static*/ && (offCode >= 20))
price += (offCode-19)*2 * BITCOST_MULTIPLIER; /* handicap for long distance offsets, favor decompression speed */
return price;
}

/* ZSTD_buildPriceTables() :
* Full rebuild. Only needed where every statistic changes at once, which
* is ZSTD_rescaleFreqs(), i.e. once per block. */
static void ZSTD_buildPriceTables(optState_t* optPtr, int optLevel)
{
U32 c;
for (c = 0; c <= MaxLL; c++)
optPtr->litLengthPriceNoSum[c] = ZSTD_litLengthPriceNoSum(optPtr, c, optLevel);
for (c = 0; c <= MaxML; c++)
optPtr->matchLengthPriceNoSum[c] = ZSTD_matchLengthPriceNoSum(optPtr, c, optLevel);
for (c = 0; c <= MaxOff; c++)
optPtr->offCodePriceNoSum[c] = ZSTD_offCodePriceNoSum(optPtr, c, optLevel);
}

static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel)
{
if (ZSTD_compressedLiterals(optPtr))
Expand Down Expand Up @@ -258,6 +313,10 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
}

ZSTD_setBasePrices(optPtr, optLevel);
/* every statistic was just replaced -> the per-code tables must be
* rebuilt wholesale. This is the only place that needs a full build;
* incremental updates in ZSTD_updateStats() keep them current after. */
ZSTD_buildPriceTables(optPtr, optLevel);
}

/* ZSTD_rawLiteralsCost() :
Expand Down Expand Up @@ -308,54 +367,83 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP

/* dynamic statistics */
{ U32 const llCode = ZSTD_LLcode(litLength);
return (LL_bits[llCode] * BITCOST_MULTIPLIER)
+ optPtr->litLengthSumBasePrice
- WEIGHT(optPtr->litLengthFreq[llCode], optLevel);
(void)optLevel;
return optPtr->litLengthSumBasePrice + optPtr->litLengthPriceNoSum[llCode];
}
}

/* ZSTD_getMatchPrice() :
* Provides the cost of the match part (offset + matchLength) of a sequence.
* Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence.
* @offBase : sumtype, representing an offset or a repcode, and using numeric representation of ZSTD_storeSeq()
* @optLevel: when <2, favors small offset for decompression speed (improved cache efficiency)
/* Cost of the match part (offset + matchLength) of a sequence.
* Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a
* sequence.
* @offBase : sumtype, representing an offset or a repcode, and using numeric
* representation of ZSTD_storeSeq()
* @optLevel: when <2, favors small offset for decompression speed (improved
* cache efficiency)
*
* The price is provided in two halves rather than as one function, because
* it splits exactly: one half depends only on the offset, the other only on
* the match length. That matters because the parser scans many candidate
* lengths against one fixed offset, so the offset half is loop-invariant
* there -- and the compiler cannot hoist it on our behalf, since
* `optPtr->offCodeFreq[]` and the `opt[].price` the loop stores to are both
* U32 lvalues and type-based alias analysis cannot prove the store does not
* clobber the load.
*
* Splitting is value-preserving: U32 addition is associative, so summing the
* halves reproduces the original sum bit for bit, and the parse is
* unaffected. A caller that wants the whole price simply adds all three:
* ZSTD_matchPriceOffPart() + ZSTD_matchPriceMLSum() + ZSTD_matchPriceMLPartNoSum()
*/

/* offset-dependent half of a match price; invariant across match lengths */
FORCE_INLINE_TEMPLATE U32
ZSTD_getMatchPrice(U32 const offBase,
U32 const matchLength,
const optState_t* const optPtr,
int const optLevel)
ZSTD_matchPriceOffPart(U32 const offBase,
const optState_t* const optPtr,
int const optLevel)
{
U32 price;
U32 const offCode = ZSTD_highbit32(offBase);
U32 const mlBase = matchLength - MINMATCH;
assert(matchLength >= MINMATCH);

(void)optLevel;
if (optPtr->priceType == zop_predef) /* fixed scheme, does not use statistics */
return WEIGHT(mlBase, optLevel)
+ ((16 + offCode) * BITCOST_MULTIPLIER); /* emulated offset cost */

/* dynamic statistics */
price = (offCode * BITCOST_MULTIPLIER) + (optPtr->offCodeSumBasePrice - WEIGHT(optPtr->offCodeFreq[offCode], optLevel));
if ((optLevel<2) /*static*/ && offCode >= 20)
price += (offCode-19)*2 * BITCOST_MULTIPLIER; /* handicap for long distance offsets, favor decompression speed */

/* match Length */
{ U32 const mlCode = ZSTD_MLcode(mlBase);
price += (ML_bits[mlCode] * BITCOST_MULTIPLIER) + (optPtr->matchLengthSumBasePrice - WEIGHT(optPtr->matchLengthFreq[mlCode], optLevel));
}
return (16 + offCode) * BITCOST_MULTIPLIER; /* emulated offset cost */
return optPtr->offCodeSumBasePrice + optPtr->offCodePriceNoSum[offCode];
}

price += BITCOST_MULTIPLIER / 5; /* heuristic : make matches a bit more costly to favor less sequences -> faster decompression speed */
/* matchLength-dependent half of a match price, EXCLUDING the
* matchLengthSumBasePrice term, which the caller adds via
* ZSTD_matchPriceMLSum(). Keeping the sum term out lets the caller hoist
* it: it does not vary across candidate lengths, but a load of it placed
* inside the loop would be reloaded on every iteration for the same
* aliasing reason as above.
* @predef must equal (optPtr->priceType == zop_predef); it too is passed
* in rather than re-read so the test can be hoisted out of the loop. */
FORCE_INLINE_TEMPLATE U32
ZSTD_matchPriceMLPartNoSum(U32 const matchLength,
const optState_t* const optPtr,
int const predef,
int const optLevel)
{
U32 const mlBase = matchLength - MINMATCH;
assert(matchLength >= MINMATCH);
if (predef) /* fixed scheme, does not use statistics */
return WEIGHT(mlBase, optLevel);
(void)optLevel;
return optPtr->matchLengthPriceNoSum[ZSTD_MLcode(mlBase)];
}

DEBUGLOG(8, "ZSTD_getMatchPrice(ml:%u) = %u", matchLength, price);
return price;
/* the term that completes ZSTD_matchPriceMLPartNoSum(); zero under the
* predefined scheme, which uses no statistics */
FORCE_INLINE_TEMPLATE U32
ZSTD_matchPriceMLSum(const optState_t* const optPtr, int const predef)
{
return predef ? 0 : optPtr->matchLengthSumBasePrice;
}

/* ZSTD_updateStats() :
* assumption : literals + litLength <= iend */
static void ZSTD_updateStats(optState_t* const optPtr,
U32 litLength, const BYTE* literals,
U32 offBase, U32 matchLength)
U32 offBase, U32 matchLength,
int const optLevel)
{
/* literals */
if (ZSTD_compressedLiterals(optPtr)) {
Expand All @@ -365,24 +453,34 @@ static void ZSTD_updateStats(optState_t* const optPtr,
optPtr->litSum += litLength*ZSTD_LITFREQ_ADD;
}

/* Each of the three sequence symbols below changes exactly one
* frequency, so exactly one entry of the matching per-code price table
* goes stale. Refreshing it here -- three evaluations per sequence --
* is what removes the need to rebuild all 121 entries after every
* stretch group. The sum terms are excluded from the tables precisely
* so that a change to a running total costs nothing here. */

/* literal Length */
{ U32 const llCode = ZSTD_LLcode(litLength);
optPtr->litLengthFreq[llCode]++;
optPtr->litLengthSum++;
optPtr->litLengthPriceNoSum[llCode] = ZSTD_litLengthPriceNoSum(optPtr, llCode, optLevel);
}

/* offset code : follows storeSeq() numeric representation */
{ U32 const offCode = ZSTD_highbit32(offBase);
assert(offCode <= MaxOff);
optPtr->offCodeFreq[offCode]++;
optPtr->offCodeSum++;
optPtr->offCodePriceNoSum[offCode] = ZSTD_offCodePriceNoSum(optPtr, offCode, optLevel);
}

/* match Length */
{ U32 const mlBase = matchLength - MINMATCH;
U32 const mlCode = ZSTD_MLcode(mlBase);
optPtr->matchLengthFreq[mlCode]++;
optPtr->matchLengthSum++;
optPtr->matchLengthPriceNoSum[mlCode] = ZSTD_matchLengthPriceNoSum(optPtr, mlCode, optLevel);
}
}

Expand Down Expand Up @@ -1101,6 +1199,12 @@ ZSTD_compressBlock_opt_generic(ZSTD_MatchState_t* ms,
ZSTD_match_t* const matches = optStatePtr->matchTable;
ZSTD_optimal_t lastStretch;
ZSTD_optLdm_t optLdm;
/* ZSTD_rescaleFreqs() fixes priceType for the whole block and nothing
* changes it afterwards, so the predefined-vs-dynamic test is hoisted
* here once instead of being re-read inside the price loops -- where
* it would alias the int the loop stores into opt[].price and so be
* reloaded on every iteration. */
int predefPrices;

ZSTD_memset(&lastStretch, 0, sizeof(ZSTD_optimal_t));

Expand All @@ -1113,6 +1217,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_MatchState_t* ms,
(U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate);
assert(optLevel <= 2);
ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel);
predefPrices = (optStatePtr->priceType == zop_predef);
ip += (ip==prefixStart);

/* Match Loop */
Expand Down Expand Up @@ -1180,8 +1285,13 @@ ZSTD_compressBlock_opt_generic(ZSTD_MatchState_t* ms,
for (matchNb = 0; matchNb < nbMatches; matchNb++) {
U32 const offBase = matches[matchNb].off;
U32 const end = matches[matchNb].len;
/* everything except the match length is the same for
* every length of this match: evaluate it once */
U32 const offPrice = ZSTD_matchPriceOffPart(offBase, optStatePtr, optLevel)
+ ZSTD_matchPriceMLSum(optStatePtr, predefPrices);
for ( ; pos <= end ; pos++ ) {
int const matchPrice = (int)ZSTD_getMatchPrice(offBase, pos, optStatePtr, optLevel);
int const matchPrice = (int)(offPrice
+ ZSTD_matchPriceMLPartNoSum(pos, optStatePtr, predefPrices, optLevel));
int const sequencePrice = opt[0].price + matchPrice;
DEBUGLOG(7, "rPos:%u => set initial price : %.2f",
pos, ZSTD_fCost(sequencePrice));
Expand Down Expand Up @@ -1306,14 +1416,22 @@ ZSTD_compressBlock_opt_generic(ZSTD_MatchState_t* ms,
U32 const offset = matches[matchNb].off;
U32 const lastML = matches[matchNb].len;
U32 const startML = (matchNb>0) ? matches[matchNb-1].len+1 : minMatch;
/* One offset, many candidate lengths. Everything in the
* price except the match-length term is loop-invariant
* here, so it is folded into one value now rather than
* re-derived on every iteration. */
U32 const offBasePrice = (U32)basePrice
+ ZSTD_matchPriceOffPart(offset, optStatePtr, optLevel)
+ ZSTD_matchPriceMLSum(optStatePtr, predefPrices);
U32 mlen;

DEBUGLOG(7, "testing match %u => offBase=%4u, mlen=%2u, llen=%2u",
matchNb, matches[matchNb].off, lastML, opt[cur].litlen);

for (mlen = lastML; mlen >= startML; mlen--) { /* scan downward */
U32 const pos = cur + mlen;
int const price = basePrice + (int)ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel);
int const price = (int)(offBasePrice
+ ZSTD_matchPriceMLPartNoSum(mlen, optStatePtr, predefPrices, optLevel));

if ((pos > last_pos) || (price < opt[pos].price)) {
DEBUGLOG(7, "rPos:%u (ml=%2u) => new better price (%.2f<%.2f)",
Expand Down Expand Up @@ -1417,7 +1535,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_MatchState_t* ms,
}

assert(anchor + llen <= iend);
ZSTD_updateStats(optStatePtr, llen, anchor, offBase, mlen);
ZSTD_updateStats(optStatePtr, llen, anchor, offBase, mlen, optLevel);
ZSTD_storeSeq(seqStore, llen, anchor, iend, offBase, mlen);
anchor += advance;
ip = anchor;
Expand Down