diff --git a/evcache-core/build.gradle b/evcache-core/build.gradle index 63874384..938137e3 100644 --- a/evcache-core/build.gradle +++ b/evcache-core/build.gradle @@ -43,6 +43,7 @@ dependencies { api group:"joda-time", name:"joda-time", version:"latest.release" api group:"javax.annotation", name:"javax.annotation-api", version:"latest.release" api group:"com.github.fzakaria", name:"ascii85", version:"latest.release" + api group:"com.github.luben", name:"zstd-jni", version:"1.5.7-11" testImplementation group:"org.testng", name:"testng", version:"7.5" testImplementation group:"com.beust", name:"jcommander", version:"1.72" diff --git a/evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java b/evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java index b4183b06..c0f31dce 100644 --- a/evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java +++ b/evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java @@ -22,25 +22,35 @@ package com.netflix.evcache; +import com.github.luben.zstd.Zstd; +import com.github.luben.zstd.ZstdInputStream; import com.netflix.evcache.config.EVCacheTranscoderProperties; +import com.netflix.evcache.config.EVCacheTranscoderProperties.CompressionAlgorithm; import com.netflix.evcache.metrics.EVCacheMetricsFactory; import com.netflix.evcache.pool.EVCacheValue; import com.netflix.evcache.pool.EVCacheValueSerde; import com.netflix.evcache.util.EVCacheConfig; import com.netflix.spectator.api.BasicTag; +import com.netflix.spectator.api.DistributionSummary; import com.netflix.spectator.api.Tag; -import com.netflix.spectator.api.Timer; import net.spy.memcached.CachedData; import net.spy.memcached.transcoders.BaseSerializingTranscoder; import net.spy.memcached.transcoders.Transcoder; import net.spy.memcached.transcoders.TranscoderUtils; -import java.time.Duration; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.ArrayList; import java.util.Date; +import java.util.EnumMap; import java.util.List; -import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Transcoder that serializes and compresses objects. @@ -48,6 +58,8 @@ public class EVCacheSerializingTranscoder extends BaseSerializingTranscoder implements Transcoder { + private static final Logger log = LoggerFactory.getLogger(EVCacheSerializingTranscoder.class); + // General flags static final int SERIALIZED = 1; static final int COMPRESSED = 2; @@ -63,12 +75,15 @@ public class EVCacheSerializingTranscoder extends BaseSerializingTranscoder impl static final int SPECIAL_DOUBLE = (7 << 8); static final int SPECIAL_BYTEARRAY = (8 << 8); - static final String COMPRESSION = "COMPRESSION_METRIC"; + public static final int DEFAULT_ZSTD_COMPRESSION_LEVEL = 3; + + private static final int ZSTD_MAGIC = 0xFD2FB528; private final TranscoderUtils tu = new TranscoderUtils(true); - private Timer timer; + protected final String appName; + protected EVCacheTranscoderProperties transcoderProperties; - protected final EVCacheTranscoderProperties transcoderProperties; + private final EnumMap compressionRatioSummaries; /** * Get a serializing transcoder with the default max data size. @@ -94,7 +109,22 @@ public EVCacheSerializingTranscoder(int max) { */ public EVCacheSerializingTranscoder(int max, EVCacheTranscoderProperties properties) { super(max); + this.appName = properties.getAppName(); this.transcoderProperties = properties; + this.compressionRatioSummaries = buildCompressionRatioSummaries(appName); + } + + private static EnumMap buildCompressionRatioSummaries(String appName) { + EnumMap summaries = new EnumMap<>(CompressionAlgorithm.class); + for (CompressionAlgorithm algo : CompressionAlgorithm.values()) { + List tagList = new ArrayList<>(2); + tagList.add(new BasicTag(EVCacheMetricsFactory.COMPRESSION_TYPE, algo.name().toLowerCase())); + if (appName != null && !appName.isEmpty()) { + tagList.add(new BasicTag(EVCacheMetricsFactory.CACHE, appName)); + } + summaries.put(algo, EVCacheMetricsFactory.getInstance().getDistributionSummary(EVCacheMetricsFactory.COMPRESSION_RATIO, tagList)); + } + return summaries; } @Override @@ -146,7 +176,7 @@ public Object decode(CachedData d) { rv = data; break; default: - getLogger().warn("Undecodeable with flags %x", flags); + log.warn("Undecodeable with flags {}", Integer.toHexString(flags)); } } else { rv = decodeString(data); @@ -194,19 +224,19 @@ public CachedData encode(Object o) { } assert b != null; if (b.length > compressionThreshold) { + int originalLength = b.length; byte[] compressed = compress(b); - if (compressed.length < b.length) { - getLogger().trace("Compressed %s from %d to %d", - o.getClass().getName(), b.length, compressed.length); + if (compressed.length < originalLength) { + if (log.isTraceEnabled()) { + log.trace("Compressed {} from {} to {}", + o.getClass().getName(), originalLength, compressed.length); + } b = compressed; flags |= COMPRESSED; - } else { - getLogger().debug("Compression increased the size of %s from %d to %d", - o.getClass().getName(), b.length, compressed.length); + } else if (log.isDebugEnabled()) { + log.debug("Compression increased the size of {} from {} to {}", + o.getClass().getName(), originalLength, compressed.length); } - - long compression_ratio = Math.round((double) compressed.length / b.length * 100); - updateTimerWithCompressionRatio(compression_ratio); } return new CachedData(flags, b, getMaxSize()); } @@ -227,14 +257,89 @@ protected Object deserialize(byte[] in) { return super.deserialize(in); } - private void updateTimerWithCompressionRatio(long ratio_percentage) { - if(timer == null) { - final List tagList = new ArrayList(1); - tagList.add(new BasicTag(EVCacheMetricsFactory.COMPRESSION_TYPE, "gzip")); - timer = EVCacheMetricsFactory.getInstance().getPercentileTimer(EVCacheMetricsFactory.COMPRESSION_RATIO, tagList, Duration.ofMillis(100)); - }; + @Override + protected byte[] compress(byte[] in) { + if (in == null) throw new NullPointerException("Can't compress null"); + + CompressionAlgorithm compressionAlgorithm = transcoderProperties.getCompressionAlgorithmProperty().get(); + byte[] compressed; + switch (compressionAlgorithm) { + case ZSTD: + int zstdLevel = transcoderProperties.getZstdCompressionLevelProperty().get(); + if (log.isDebugEnabled()) { + log.debug("algorithm: {}, level: {}, appName: {}", compressionAlgorithm, zstdLevel, appName); + } + compressed = Zstd.compress(in, zstdLevel); + break; + case GZIP: + if (log.isDebugEnabled()) { + log.debug("algorithm: {}, appName: {}", compressionAlgorithm, appName); + } + compressed = super.compress(in); + break; + default: + throw new IllegalArgumentException("Unsupported compression algorithm: " + compressionAlgorithm); + } - timer.record(ratio_percentage, TimeUnit.MILLISECONDS); + if (compressed != null) { + long ratioPerCent = Math.round((double) compressed.length / in.length * 100.0); + recordCompressionRatio(ratioPerCent, compressionAlgorithm); + } + + return compressed; } + @Override + protected byte[] decompress(byte[] in) { + if (in == null || in.length == 0) return in; + if (isZstdCompressed(in)) return decompressZstd(in); + return super.decompress(in); + } + + private boolean isZstdCompressed(byte[] data) { + if (data == null || data.length < 4) return false; + int magic = ByteBuffer.wrap(data, 0, 4).order(ByteOrder.LITTLE_ENDIAN).getInt(); + return magic == ZSTD_MAGIC; + } + + private byte[] decompressZstd(byte[] in) { + long originalSize = Zstd.getFrameContentSize(in); + if (originalSize > Integer.MAX_VALUE) { + throw new RuntimeException("Zstd decompressed size exceeds int range: " + originalSize); + } + if (originalSize > 0) { + // Fast path: frame carries a content-size header (compress() above always does). + return Zstd.decompress(in, (int) originalSize); + } + // Slow path: declared size is 0, unknown (-1), or invalid (-2) — stream-decode and let + // ZstdInputStream surface any frame errors. + log.warn("Zstd frame missing content-size header (getFrameContentSize={}); falling back to stream decode. appName={}", originalSize, appName); + ZstdInputStream zis = null; + try { + zis = new ZstdInputStream(new ByteArrayInputStream(in)); + return readAll(zis); + } catch (IOException e) { + log.error("Error reading Zstd input stream", e); + return null; + } finally { + try { if (zis != null) zis.close(); } catch (IOException ignored) {} + } + } + + private static byte[] readAll(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[8192]; + int n; + while ((n = in.read(buf)) != -1) { + out.write(buf, 0, n); + } + return out.toByteArray(); + } + + private void recordCompressionRatio(long ratioPerCent, CompressionAlgorithm compressionAlgorithm) { + DistributionSummary summary = compressionRatioSummaries.get(compressionAlgorithm); + if (summary != null) { + summary.record(ratioPerCent); + } + } } diff --git a/evcache-core/src/main/java/com/netflix/evcache/config/EVCacheTranscoderProperties.java b/evcache-core/src/main/java/com/netflix/evcache/config/EVCacheTranscoderProperties.java index b7ceb631..e0d747ee 100644 --- a/evcache-core/src/main/java/com/netflix/evcache/config/EVCacheTranscoderProperties.java +++ b/evcache-core/src/main/java/com/netflix/evcache/config/EVCacheTranscoderProperties.java @@ -2,6 +2,7 @@ import com.netflix.archaius.api.Property; import com.netflix.archaius.api.PropertyRepository; +import net.spy.memcached.compat.SpyObject; /** * Properties related to {@link com.netflix.evcache.EVCacheTranscoder} @@ -21,16 +22,22 @@ * {@code .get()}. * */ -public final class EVCacheTranscoderProperties { +public final class EVCacheTranscoderProperties extends SpyObject { public static final boolean DEFAULT_BINARY_SERIALIZATION_ENABLED = false; public static final int DEFAULT_MAX_DATA_SIZE_BYTES = 20 * 1024 * 1024; public static final int DEFAULT_COMPRESSION_THRESHOLD_BYTES = 120; + public static final CompressionAlgorithm DEFAULT_COMPRESSION_ALGORITHM = CompressionAlgorithm.GZIP; + public static final int DEFAULT_COMPRESSION_ZSTD_LEVEL = 3; + + public enum CompressionAlgorithm { GZIP, ZSTD } public enum Key { BINARY_SERIALIZATION_ENABLED("binary.serialization.enabled", "default.evcache.binary.serialization.enabled"), MAX_DATA_SIZE_BYTES("max.data.size", "default.evcache.max.data.size"), - COMPRESSION_THRESHOLD_BYTES("compression.threshold", "default.evcache.compression.threshold"); + COMPRESSION_THRESHOLD_BYTES("compression.threshold", "default.evcache.compression.threshold"), + COMPRESSION_ALGORITHM("compression.algorithm", "default.evcache.compression.algorithm"), + COMPRESSION_ZSTD_LEVEL("compression.zstd.level", "default.evcache.compression.zstd.level"); final String appKeySuffix; final String globalKey; @@ -46,6 +53,8 @@ public enum Key { private final boolean binarySerializationEnabled; private final int maxDataSizeBytes; private final int compressionThresholdBytes; + private final Property compressionAlgorithmProperty; + private final Property zstdCompressionLevelProperty; /** * Construct the bundle and snapshot every property via the three-level resolution chain. @@ -68,6 +77,11 @@ public EVCacheTranscoderProperties(String appName, PropertyRepository propertyRe Key.MAX_DATA_SIZE_BYTES, Integer.class, DEFAULT_MAX_DATA_SIZE_BYTES).get(); this.compressionThresholdBytes = getProperty(appName, propertyRepository, Key.COMPRESSION_THRESHOLD_BYTES, Integer.class, DEFAULT_COMPRESSION_THRESHOLD_BYTES).get(); + this.compressionAlgorithmProperty = getProperty(appName, propertyRepository, + Key.COMPRESSION_ALGORITHM, String.class, DEFAULT_COMPRESSION_ALGORITHM.name()) + .map(this::parseCompressionAlgorithm); + this.zstdCompressionLevelProperty = getProperty(appName, propertyRepository, + Key.COMPRESSION_ZSTD_LEVEL, Integer.class, DEFAULT_COMPRESSION_ZSTD_LEVEL); } public String getAppName() { @@ -86,6 +100,40 @@ public int getCompressionThresholdBytes() { return compressionThresholdBytes; } + /** + * Live-updating {@link Property} handle for the transcoder compression algorithm. Calling + * {@code .get()} returns the current {@link CompressionAlgorithm} value; underlying storage + * is a String property (case-insensitive) so ops can set the FP as {@code "gzip"} or + * {@code "ZSTD"} interchangeably. Handle is resolved once at bundle construction and + * shared across callers — {@code .get()} on it always observes the latest FP value. + */ + public Property getCompressionAlgorithmProperty() { + return compressionAlgorithmProperty; + } + + /** + * Live-updating {@link Property} handle for the zstd compression level. Resolved once at + * bundle construction. + */ + public Property getZstdCompressionLevelProperty() { + return zstdCompressionLevelProperty; + } + + /** + * Parse an FP algorithm string (case-insensitive) into a {@link CompressionAlgorithm}. An + * unrecognized value falls back to {@link #DEFAULT_COMPRESSION_ALGORITHM} rather than + * propagating a {@code null} (which would NPE the compression switch at encode time) — a + * typo'd fast property degrades to the default instead of taking down writes. + */ + private CompressionAlgorithm parseCompressionAlgorithm(String value) { + try { + return CompressionAlgorithm.valueOf(value.toUpperCase()); + } catch (IllegalArgumentException | NullPointerException e) { + getLogger().warn("Unrecognized compression algorithm '%s'; falling back to %s", value, DEFAULT_COMPRESSION_ALGORITHM); + return DEFAULT_COMPRESSION_ALGORITHM; + } + } + private static Property getProperty(String appName, PropertyRepository propertyRepository, Key key, Class type, T defaultValue) { if (appName == null || appName.isEmpty()) { diff --git a/evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java b/evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java new file mode 100644 index 00000000..03ef8441 --- /dev/null +++ b/evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java @@ -0,0 +1,333 @@ +package com.netflix.evcache; + +import com.netflix.archaius.DefaultPropertyFactory; +import com.netflix.archaius.config.DefaultSettableConfig; +import com.netflix.evcache.config.EVCacheTranscoderProperties; +import com.netflix.evcache.config.EVCacheTranscoderProperties.CompressionAlgorithm; +import com.netflix.evcache.metrics.EVCacheMetricsFactory; +import com.netflix.spectator.api.DefaultRegistry; +import com.netflix.spectator.api.Id; +import com.netflix.spectator.api.Meter; +import com.netflix.spectator.api.Registry; +import com.netflix.spectator.api.Spectator; +import com.netflix.spectator.api.Tag; +import net.spy.memcached.CachedData; +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + +public class EVCacheSerializingTranscoderTest { + + /** + * Build a serializing transcoder whose compression algorithm/level are resolved from a + * fresh {@link EVCacheTranscoderProperties} bundle (global keys, no app prefix). + */ + private EVCacheSerializingTranscoder buildTranscoder(String algo, Integer level) { + DefaultSettableConfig config = new DefaultSettableConfig(); + if (algo != null) config.setProperty("default.evcache.compression.algorithm", algo); + if (level != null) config.setProperty("default.evcache.compression.zstd.level", level); + EVCacheTranscoderProperties props = + new EVCacheTranscoderProperties(null, new DefaultPropertyFactory(config)); + return new EVCacheSerializingTranscoder(CachedData.MAX_SIZE, props); + } + + /** + * Build an {@link EVCacheTranscoder} from a config, resolved for the given app name, with the + * compression threshold forced low so the short test payloads always compress. + */ + private EVCacheTranscoder buildEVCacheTranscoder(String appName, DefaultSettableConfig config, int threshold) { + EVCacheTranscoder t = + new EVCacheTranscoder(new EVCacheTranscoderProperties(appName, new DefaultPropertyFactory(config))); + t.setCompressionThreshold(threshold); + return t; + } + + @Test + public void testEnumValues() { + assertEquals(CompressionAlgorithm.valueOf("GZIP"), CompressionAlgorithm.GZIP); + assertEquals(CompressionAlgorithm.valueOf("ZSTD"), CompressionAlgorithm.ZSTD); + } + + @Test + public void testDefaultZstdLevelConstant() { + assertEquals(EVCacheSerializingTranscoder.DEFAULT_ZSTD_COMPRESSION_LEVEL, 3); + } + + @Test + public void testCustomZstdLevelRoundTrip() { + EVCacheSerializingTranscoder t = buildTranscoder("ZSTD", 5); + t.setCompressionThreshold(1); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = t.encode(original); + String decoded = (String) t.decode(encoded); + assertEquals(decoded, original, "Round-trip must succeed with custom zstd level 5"); + } + + @Test + public void testGzipEncodeSetsGzipMagicBytes() { + EVCacheSerializingTranscoder t = buildTranscoder("GZIP", null); + t.setCompressionThreshold(0); + CachedData encoded = t.encode("hello world hello world hello world hello world hello world"); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, + "COMPRESSED flag must be set"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x1f, "Expected gzip magic byte 0"); + assertEquals(data[1], (byte) 0x8b, "Expected gzip magic byte 1"); + } + + @Test + public void testZstdEncodeSetsZstdMagicBytes() { + EVCacheSerializingTranscoder t = buildTranscoder("ZSTD", null); + t.setCompressionThreshold(0); + CachedData encoded = t.encode("hello world hello world hello world hello world hello world"); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, + "COMPRESSED flag must be set"); + byte[] data = encoded.getData(); + // Zstd magic is 0xFD2FB528 in little-endian: bytes 0x28 0xB5 0x2F 0xFD + assertEquals(data[0], (byte) 0x28, "Expected zstd magic byte 0"); + assertEquals(data[1], (byte) 0xB5, "Expected zstd magic byte 1"); + assertEquals(data[2], (byte) 0x2F, "Expected zstd magic byte 2"); + assertEquals(data[3], (byte) 0xFD, "Expected zstd magic byte 3"); + } + + @Test + public void testCompressionThatGrowsDataLeavesCompressedFlagUnset() { + // encode() only sets COMPRESSED (and keeps the compressed bytes) when compression actually + // shrinks the payload. Tiny incompressible input grows under gzip framing, so the original + // bytes must be kept and the COMPRESSED flag must stay clear. + EVCacheSerializingTranscoder t = buildTranscoder("GZIP", null); + t.setCompressionThreshold(0); + byte[] tiny = new byte[] {1, 2, 3, 4, 5, 6, 7, 8}; + CachedData encoded = t.encode(tiny); + assertEquals(encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED, 0, + "COMPRESSED flag must not be set when compression grows the payload"); + assertEquals(encoded.getData(), tiny, "original bytes must be kept when compression does not help"); + assertEquals((byte[]) t.decode(encoded), tiny, "round-trip must return the original bytes"); + } + + @Test + public void testGzipRoundTrip() { + EVCacheSerializingTranscoder transcoder = buildTranscoder("GZIP", null); + transcoder.setCompressionThreshold(1); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = transcoder.encode(original); + String decoded = (String) transcoder.decode(encoded); + assertEquals(decoded, original); + } + + @Test + public void testZstdRoundTrip() { + EVCacheSerializingTranscoder transcoder = buildTranscoder("ZSTD", null); + transcoder.setCompressionThreshold(1); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = transcoder.encode(original); + String decoded = (String) transcoder.decode(encoded); + assertEquals(decoded, original); + } + + @Test + public void testGzipTranscoderDecodesZstdData() { + // zstd transcoder writes, gzip transcoder reads → cross-decode via magic-byte detection + EVCacheSerializingTranscoder writer = buildTranscoder("ZSTD", null); + writer.setCompressionThreshold(1); + EVCacheSerializingTranscoder reader = buildTranscoder("GZIP", null); + + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = writer.encode(original); + String decoded = (String) reader.decode(encoded); + assertEquals(decoded, original); + } + + @Test + public void testZstdTranscoderDecodesGzipData() { + // gzip transcoder writes, zstd transcoder reads → cross-decode via magic-byte detection + EVCacheSerializingTranscoder writer = buildTranscoder("GZIP", null); + writer.setCompressionThreshold(1); + EVCacheSerializingTranscoder reader = buildTranscoder("ZSTD", null); + + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = writer.encode(original); + String decoded = (String) reader.decode(encoded); + assertEquals(decoded, original); + } + + @Test + public void testEVCacheTranscoderDefaultsToGzip() { + EVCacheTranscoder transcoder = buildEVCacheTranscoder(null, new DefaultSettableConfig(), 0); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = transcoder.encode(original); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, + "COMPRESSED flag must be set"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x1f, "EVCacheTranscoder must default to gzip"); + assertEquals(data[1], (byte) 0x8b, "EVCacheTranscoder must default to gzip"); + String decoded = (String) transcoder.decode(encoded); + assertEquals(decoded, original); + } + + @Test + public void testEVCacheTranscoderExplicitZstdAlgorithm() { + DefaultSettableConfig config = new DefaultSettableConfig(); + config.setProperty("default.evcache.compression.algorithm", "ZSTD"); + config.setProperty("default.evcache.compression.zstd.level", + EVCacheSerializingTranscoder.DEFAULT_ZSTD_COMPRESSION_LEVEL); + EVCacheTranscoder transcoder = buildEVCacheTranscoder(null, config, 1); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = transcoder.encode(original); + String decoded = (String) transcoder.decode(encoded); + assertEquals(decoded, original); + } + + @Test + public void testAppNamePrefixedAlgoOverridesDefault() { + DefaultSettableConfig config = new DefaultSettableConfig(); + config.setProperty("default.evcache.compression.algorithm", "GZIP"); + config.setProperty("EVCACHE_TEST.compression.algorithm", "ZSTD"); + EVCacheTranscoder transcoder = buildEVCacheTranscoder("EVCACHE_TEST", config, 1); + CachedData encoded = transcoder.encode("hello world hello world hello world hello world hello world"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x28, "app-specific ZSTD override must win over default GZIP"); + assertEquals(data[1], (byte) 0xB5, "app-specific ZSTD override must win over default GZIP"); + } + + @Test + public void testAppNameFallsBackToDefaultAlgoWhenNoOverride() { + DefaultSettableConfig config = new DefaultSettableConfig(); + config.setProperty("default.evcache.compression.algorithm", "ZSTD"); + EVCacheTranscoder transcoder = buildEVCacheTranscoder("EVCACHE_NO_OVERRIDE", config, 1); + CachedData encoded = transcoder.encode("hello world hello world hello world hello world hello world"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x28, "must fall back to default ZSTD when no app-specific override exists"); + assertEquals(data[1], (byte) 0xB5, "must fall back to default ZSTD when no app-specific override exists"); + } + + @Test + public void testAppNamePrefixedZstdLevelRoundTrip() { + DefaultSettableConfig config = new DefaultSettableConfig(); + config.setProperty("default.evcache.compression.algorithm", "ZSTD"); + config.setProperty("default.evcache.compression.zstd.level", 1); + config.setProperty("EVCACHE_TEST.compression.zstd.level", 5); + EVCacheTranscoder transcoder = buildEVCacheTranscoder("EVCACHE_TEST", config, 1); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = transcoder.encode(original); + String decoded = (String) transcoder.decode(encoded); + assertEquals(decoded, original, "app-specific zstd level override round-trip must succeed"); + } + + @Test + public void testCompressionRatioMetricTaggedWithAppName() { + final String appName = "EVCACHE_RATIO_TEST"; + Registry registry = new DefaultRegistry(); + Spectator.globalRegistry().add(registry); + try { + DefaultSettableConfig config = new DefaultSettableConfig(); + config.setProperty("default.evcache.compression.algorithm", "GZIP"); + EVCacheTranscoderProperties props = + new EVCacheTranscoderProperties(appName, new DefaultPropertyFactory(config)); + EVCacheSerializingTranscoder t = new EVCacheSerializingTranscoder(CachedData.MAX_SIZE, props); + t.setCompressionThreshold(0); + t.encode("hello world hello world hello world hello world hello world"); + + assertTrue(hasCompressionRatioCacheTag(registry, appName), + "compression ratio metric must carry the " + EVCacheMetricsFactory.CACHE + " tag with the app name"); + } finally { + Spectator.globalRegistry().remove(registry); + } + } + + @Test + public void testCompressionRatioMetricNotTaggedWhenNoAppName() { + Registry registry = new DefaultRegistry(); + Spectator.globalRegistry().add(registry); + try { + EVCacheSerializingTranscoder t = buildTranscoder("GZIP", null); + t.setCompressionThreshold(0); + t.encode("hello world hello world hello world hello world hello world"); + + for (Meter meter : registry) { + Id id = meter.id(); + if (EVCacheMetricsFactory.COMPRESSION_RATIO.equals(id.name())) { + for (Tag tag : id.tags()) { + assertNotEquals(tag.key(), EVCacheMetricsFactory.CACHE, + "no app name tag must be added when app name is absent"); + } + } + } + } finally { + Spectator.globalRegistry().remove(registry); + } + } + + private boolean hasCompressionRatioCacheTag(Registry registry, String appName) { + for (Meter meter : registry) { + Id id = meter.id(); + if (EVCacheMetricsFactory.COMPRESSION_RATIO.equals(id.name())) { + for (Tag tag : id.tags()) { + if (EVCacheMetricsFactory.CACHE.equals(tag.key()) && appName.equals(tag.value())) { + return true; + } + } + } + } + return false; + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testInvalidAlgorithmEnumThrows() { + CompressionAlgorithm.valueOf("INVALID"); + } + + @Test + public void testFPAlgorithmGzip() { + DefaultSettableConfig config = new DefaultSettableConfig(); + config.setProperty("default.evcache.compression.algorithm", "GZIP"); + EVCacheTranscoder transcoder = buildEVCacheTranscoder(null, config, 1); + CachedData encoded = transcoder.encode("hello world hello world hello world hello world hello world"); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, + "COMPRESSED flag must be set"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x1f, "FP GZIP must produce gzip magic byte 0"); + assertEquals(data[1], (byte) 0x8b, "FP GZIP must produce gzip magic byte 1"); + } + + @Test + public void testFPAlgorithmZstd() { + DefaultSettableConfig config = new DefaultSettableConfig(); + config.setProperty("default.evcache.compression.algorithm", "ZSTD"); + EVCacheTranscoder transcoder = buildEVCacheTranscoder(null, config, 1); + CachedData encoded = transcoder.encode("hello world hello world hello world hello world hello world"); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, + "COMPRESSED flag must be set"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x28, "FP ZSTD must produce zstd magic byte 0"); + assertEquals(data[1], (byte) 0xB5, "FP ZSTD must produce zstd magic byte 1"); + } + + @Test + public void testUnrecognizedAlgorithmFallsBackToGzipAndEncodes() { + // An unknown FP algorithm value must not NPE encode(); it degrades to the default (GZIP). + DefaultSettableConfig config = new DefaultSettableConfig(); + config.setProperty("default.evcache.compression.algorithm", "SNAPPY"); + EVCacheTranscoder transcoder = buildEVCacheTranscoder(null, config, 1); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = transcoder.encode(original); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, + "COMPRESSED flag must be set"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x1f, "unknown algorithm must fall back to gzip magic byte 0"); + assertEquals(data[1], (byte) 0x8b, "unknown algorithm must fall back to gzip magic byte 1"); + assertEquals((String) transcoder.decode(encoded), original, "round-trip must succeed after fallback"); + } + + @Test + public void testFPZstdLevel() { + DefaultSettableConfig config = new DefaultSettableConfig(); + config.setProperty("default.evcache.compression.algorithm", "ZSTD"); + config.setProperty("default.evcache.compression.zstd.level", 1); + EVCacheTranscoder transcoder = buildEVCacheTranscoder(null, config, 1); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = transcoder.encode(original); + String decoded = (String) transcoder.decode(encoded); + assertEquals(decoded, original, "FP zstd level 1 round-trip must succeed"); + } +} diff --git a/evcache-core/src/test/java/com/netflix/evcache/config/EVCacheTranscoderPropertiesTest.java b/evcache-core/src/test/java/com/netflix/evcache/config/EVCacheTranscoderPropertiesTest.java index 619e8b44..30078dd8 100644 --- a/evcache-core/src/test/java/com/netflix/evcache/config/EVCacheTranscoderPropertiesTest.java +++ b/evcache-core/src/test/java/com/netflix/evcache/config/EVCacheTranscoderPropertiesTest.java @@ -1,10 +1,15 @@ package com.netflix.evcache.config; +import static com.netflix.evcache.config.EVCacheTranscoderProperties.DEFAULT_COMPRESSION_ALGORITHM; +import static com.netflix.evcache.config.EVCacheTranscoderProperties.DEFAULT_COMPRESSION_THRESHOLD_BYTES; +import static com.netflix.evcache.config.EVCacheTranscoderProperties.DEFAULT_COMPRESSION_ZSTD_LEVEL; +import static com.netflix.evcache.config.EVCacheTranscoderProperties.DEFAULT_MAX_DATA_SIZE_BYTES; import static org.assertj.core.api.Assertions.assertThat; import com.netflix.archaius.DefaultPropertyFactory; import com.netflix.archaius.api.PropertyRepository; import com.netflix.archaius.config.DefaultSettableConfig; +import com.netflix.evcache.config.EVCacheTranscoderProperties.CompressionAlgorithm; import org.testng.annotations.Test; @@ -23,6 +28,10 @@ public class EVCacheTranscoderPropertiesTest { private static final String MAX_DATA_SIZE_GLOBAL_KEY = "default.evcache.max.data.size"; private static final String COMPRESSION_PER_APP_KEY = "MYAPP.compression.threshold"; private static final String COMPRESSION_GLOBAL_KEY = "default.evcache.compression.threshold"; + private static final String ALGORITHM_PER_APP_KEY = "MYAPP.compression.algorithm"; + private static final String ALGORITHM_GLOBAL_KEY = "default.evcache.compression.algorithm"; + private static final String ZSTD_LEVEL_PER_APP_KEY = "MYAPP.compression.zstd.level"; + private static final String ZSTD_LEVEL_GLOBAL_KEY = "default.evcache.compression.zstd.level"; private static PropertyRepository repo(DefaultSettableConfig cfg) { return DefaultPropertyFactory.from(cfg); @@ -96,7 +105,7 @@ public void maxDataSize_globalFallbackWhenPerAppUnset() { @Test public void maxDataSize_staticDefaultWhenBothUnset() { EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(APP, repo(new DefaultSettableConfig())); - assertThat(props.getMaxDataSizeBytes()).isEqualTo(EVCacheTranscoderProperties.DEFAULT_MAX_DATA_SIZE_BYTES); + assertThat(props.getMaxDataSizeBytes()).isEqualTo(DEFAULT_MAX_DATA_SIZE_BYTES); } @Test @@ -141,7 +150,7 @@ public void compressionThreshold_globalFallbackWhenPerAppUnset() { @Test public void compressionThreshold_staticDefaultWhenBothUnset() { EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(APP, repo(new DefaultSettableConfig())); - assertThat(props.getCompressionThresholdBytes()).isEqualTo(EVCacheTranscoderProperties.DEFAULT_COMPRESSION_THRESHOLD_BYTES); + assertThat(props.getCompressionThresholdBytes()).isEqualTo(DEFAULT_COMPRESSION_THRESHOLD_BYTES); } @Test @@ -163,6 +172,130 @@ public void compressionThreshold_nullAppNameUsesGlobalKey() { assertThat(props.getCompressionThresholdBytes()).isEqualTo(512); } + // ---- COMPRESSION_ALGORITHM ---- + + @Test + public void compressionAlgorithm_perAppOverrideWins() { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(ALGORITHM_PER_APP_KEY, "ZSTD"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(APP, repo(cfg)); + assertThat(props.getCompressionAlgorithmProperty().get()).isEqualTo(CompressionAlgorithm.ZSTD); + } + + @Test + public void compressionAlgorithm_globalFallbackWhenPerAppUnset() { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(ALGORITHM_GLOBAL_KEY, "ZSTD"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(APP, repo(cfg)); + assertThat(props.getCompressionAlgorithmProperty().get()).isEqualTo(CompressionAlgorithm.ZSTD); + } + + @Test + public void compressionAlgorithm_staticDefaultWhenBothUnset() { + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(APP, repo(new DefaultSettableConfig())); + assertThat(props.getCompressionAlgorithmProperty().get()).isEqualTo(DEFAULT_COMPRESSION_ALGORITHM); + } + + @Test + public void compressionAlgorithm_perAppBeatsGlobal() { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(ALGORITHM_PER_APP_KEY, "ZSTD"); + cfg.setProperty(ALGORITHM_GLOBAL_KEY, "GZIP"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(APP, repo(cfg)); + assertThat(props.getCompressionAlgorithmProperty().get()).isEqualTo(CompressionAlgorithm.ZSTD); + } + + @Test + public void compressionAlgorithm_caseInsensitive() { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(ALGORITHM_GLOBAL_KEY, "zstd"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(null, repo(cfg)); + assertThat(props.getCompressionAlgorithmProperty().get()).isEqualTo(CompressionAlgorithm.ZSTD); + } + + @Test + public void compressionAlgorithm_unrecognizedValueFallsBackToDefault() { + // A typo'd/unknown FP value must degrade to the default algorithm, not resolve to null + // (which would NPE the compression switch at encode time). + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(ALGORITHM_GLOBAL_KEY, "SNAPPY"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(null, repo(cfg)); + assertThat(props.getCompressionAlgorithmProperty().get()).isEqualTo(DEFAULT_COMPRESSION_ALGORITHM); + } + + @Test + public void compressionAlgorithm_handleReflectsLiveFpUpdate() { + // getCompressionAlgorithmProperty() returns a live handle, not a snapshot: a later FP + // change must be observed by a subsequent .get() without rebuilding the bundle. + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(ALGORITHM_GLOBAL_KEY, "GZIP"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(null, repo(cfg)); + assertThat(props.getCompressionAlgorithmProperty().get()).isEqualTo(CompressionAlgorithm.GZIP); + + cfg.setProperty(ALGORITHM_GLOBAL_KEY, "ZSTD"); + assertThat(props.getCompressionAlgorithmProperty().get()).isEqualTo(CompressionAlgorithm.ZSTD); + } + + @Test + public void compressionAlgorithm_nullAppNameUsesGlobalKey() { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(ALGORITHM_GLOBAL_KEY, "ZSTD"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(null, repo(cfg)); + assertThat(props.getCompressionAlgorithmProperty().get()).isEqualTo(CompressionAlgorithm.ZSTD); + } + + // ---- COMPRESSION_ZSTD_LEVEL ---- + + @Test + public void zstdLevel_perAppOverrideWins() { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(ZSTD_LEVEL_PER_APP_KEY, "9"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(APP, repo(cfg)); + assertThat(props.getZstdCompressionLevelProperty().get()).isEqualTo(9); + } + + @Test + public void zstdLevel_globalFallbackWhenPerAppUnset() { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(ZSTD_LEVEL_GLOBAL_KEY, "9"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(APP, repo(cfg)); + assertThat(props.getZstdCompressionLevelProperty().get()).isEqualTo(9); + } + + @Test + public void zstdLevel_staticDefaultWhenBothUnset() { + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(APP, repo(new DefaultSettableConfig())); + assertThat(props.getZstdCompressionLevelProperty().get()).isEqualTo(DEFAULT_COMPRESSION_ZSTD_LEVEL); + } + + @Test + public void zstdLevel_perAppBeatsGlobal() { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(ZSTD_LEVEL_PER_APP_KEY, "5"); + cfg.setProperty(ZSTD_LEVEL_GLOBAL_KEY, "1"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(APP, repo(cfg)); + assertThat(props.getZstdCompressionLevelProperty().get()).isEqualTo(5); + } + + @Test + public void zstdLevel_nullAppNameUsesGlobalKey() { + DefaultSettableConfig cfg = new DefaultSettableConfig(); + cfg.setProperty(ZSTD_LEVEL_GLOBAL_KEY, "9"); + + EVCacheTranscoderProperties props = new EVCacheTranscoderProperties(null, repo(cfg)); + assertThat(props.getZstdCompressionLevelProperty().get()).isEqualTo(9); + } + // ---- appName ---- @Test diff --git a/evcache-core/src/test/java/test-suite.xml b/evcache-core/src/test/java/test-suite.xml index e31a08d4..39ab6875 100644 --- a/evcache-core/src/test/java/test-suite.xml +++ b/evcache-core/src/test/java/test-suite.xml @@ -12,6 +12,11 @@ + + + + +