From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Martijn Muijsers Date: Wed, 30 Nov 2022 21:51:16 +0100 Subject: [PATCH] Faster chunk serialization License: LGPL-3.0 (https://www.gnu.org/licenses/lgpl-3.0.html) Gale - https://galemc.org This patch is based on the following mixins and classes: * "net/caffeinemc/mods/lithium/common/world/chunk/CompactingPackedIntegerArray.java" * "net/caffeinemc/mods/lithium/common/world/chunk/LithiumHashPalette.java" * "net/caffeinemc/mods/lithium/mixin/chunk/serialization/SimpleBitStorageMixin.java" * "net/caffeinemc/mods/lithium/mixin/chunk/serialization/PalettedContainerMixin.java" By: Angeline As part of: Lithium (https://github.com/CaffeineMC/lithium-fabric) Licensed under: LGPL-3.0 (https://www.gnu.org/licenses/lgpl-3.0.html) diff --git a/net/minecraft/util/BitStorage.java b/net/minecraft/util/BitStorage.java index 32fe9b22e1d3a422dd80c64d61156dbc7241ba20..00e56ccb9dc2b8a6c83ba9b7e44f61deb5dc24f3 100644 --- a/net/minecraft/util/BitStorage.java +++ b/net/minecraft/util/BitStorage.java @@ -20,4 +20,6 @@ public interface BitStorage { void unpack(int[] array); BitStorage copy(); + + void compact(net.minecraft.world.level.chunk.Palette srcPalette, net.minecraft.world.level.chunk.Palette dstPalette, short[] out); // Gale - Lithium - faster chunk serialization } diff --git a/net/minecraft/util/SimpleBitStorage.java b/net/minecraft/util/SimpleBitStorage.java index 6fb3a3f167d8cbaa78135af0c180b592661e2c1d..10db8a7bfab5a305a2a3f4ff7c3cd55066e7d1a9 100644 --- a/net/minecraft/util/SimpleBitStorage.java +++ b/net/minecraft/util/SimpleBitStorage.java @@ -360,4 +360,45 @@ public class SimpleBitStorage implements BitStorage { super(message); } } + + // Gale start - Lithium - faster chunk serialization + @Override + public void compact(net.minecraft.world.level.chunk.Palette srcPalette, net.minecraft.world.level.chunk.Palette dstPalette, short[] out) { + if (this.size >= Short.MAX_VALUE) { + throw new IllegalStateException("Array too large"); + } + + if (this.size != out.length) { + throw new IllegalStateException("Array size mismatch"); + } + + short[] mappings = new short[(int) (this.mask + 1)]; + + int idx = 0; + + for (long word : this.data) { + long bits = word; + + for (int elementIdx = 0; elementIdx < this.valuesPerLong; ++elementIdx) { + int value = (int) (bits & this.mask); + int remappedId = mappings[value]; + + if (remappedId == 0) { + remappedId = dstPalette.idFor(srcPalette.valueFor(value)) + 1; + mappings[value] = (short) remappedId; + } + + out[idx] = (short) (remappedId - 1); + bits >>= this.bits; + + ++idx; + + if (idx >= this.size) { + return; + } + } + } + } + // Gale end - Lithium - faster chunk serialization + } diff --git a/net/minecraft/util/ZeroBitStorage.java b/net/minecraft/util/ZeroBitStorage.java index 64945449807e49115b9115ee114cd3acad3d1ef8..41cd253abf87807167e24a537a6e5bbe52f7daee 100644 --- a/net/minecraft/util/ZeroBitStorage.java +++ b/net/minecraft/util/ZeroBitStorage.java @@ -62,4 +62,6 @@ public class ZeroBitStorage implements BitStorage { public BitStorage copy() { return this; } + + @Override public void compact(net.minecraft.world.level.chunk.Palette srcPalette, net.minecraft.world.level.chunk.Palette dstPalette, short[] out) {} // Gale - Lithium - faster chunk serialization } diff --git a/net/minecraft/world/level/chunk/PalettedContainer.java b/net/minecraft/world/level/chunk/PalettedContainer.java index 6781df510906f4e7e51ea852f3d2f6dcbea42b3b..8ea91b6437cdc4fe53a7c5e944109a8b6f372908 100644 --- a/net/minecraft/world/level/chunk/PalettedContainer.java +++ b/net/minecraft/world/level/chunk/PalettedContainer.java @@ -25,6 +25,22 @@ import net.minecraft.util.ThreadingDetector; import net.minecraft.util.ZeroBitStorage; public class PalettedContainer implements PaletteResize, PalettedContainerRO { + + // Gale start - Lithium - faster chunk serialization + private static final ThreadLocal CACHED_ARRAY_4096 = ThreadLocal.withInitial(() -> new short[4096]); + private static final ThreadLocal CACHED_ARRAY_64 = ThreadLocal.withInitial(() -> new short[64]); + private Optional asOptional(long[] data) { + return Optional.of(Arrays.stream(data)); + } + private short[] getOrCreate(int size) { + return switch (size) { + case 64 -> CACHED_ARRAY_64.get(); + case 4096 -> CACHED_ARRAY_4096.get(); + default -> new short[size]; + }; + } + // Gale end - Lithium - faster chunk serialization + private static final int MIN_PALETTE_BITS = 0; private final PaletteResize dummyPaletteResize = (bits, objectAdded) -> 0; public final IdMap registry; @@ -307,28 +323,54 @@ public class PalettedContainer implements PaletteResize, PalettedContainer public synchronized PalettedContainerRO.PackedData pack(IdMap registry, PalettedContainer.Strategy strategy) { // Paper - synchronize this.acquire(); - PalettedContainerRO.PackedData var12; + // Gale start - Lithium - faster chunk serialization + Optional data = Optional.empty(); + List elements = null; try { - HashMapPalette hashMapPalette = new HashMapPalette<>(registry, this.data.storage.getBits(), this.dummyPaletteResize); - int size = strategy.size(); - int[] ints = new int[size]; - this.data.storage.unpack(ints); - swapPalette(ints, id -> hashMapPalette.idFor(this.data.palette.valueFor(id))); - int i = strategy.calculateBitsForSerialization(registry, hashMapPalette.getSize()); - Optional optional; - if (i != 0) { - SimpleBitStorage simpleBitStorage = new SimpleBitStorage(i, size, ints); - optional = Optional.of(Arrays.stream(simpleBitStorage.getRaw())); - } else { - optional = Optional.empty(); + // The palette that will be serialized + net.caffeinemc.mods.lithium.common.world.chunk.LithiumHashPalette hashPalette = null; + + final Palette palette = this.data.palette(); + final BitStorage storage = this.data.storage(); + if (storage instanceof ZeroBitStorage || palette.getSize() == 1) { + // If the palette only contains one entry, don't attempt to repack it. + elements = List.of(palette.valueFor(0)); + } else if (palette instanceof net.caffeinemc.mods.lithium.common.world.chunk.LithiumHashPalette lithiumHashPalette) { + hashPalette = lithiumHashPalette; } - var12 = new PalettedContainerRO.PackedData<>(hashMapPalette.getEntries(), optional); + if (elements == null) { + net.caffeinemc.mods.lithium.common.world.chunk.LithiumHashPalette compactedPalette = new net.caffeinemc.mods.lithium.common.world.chunk.LithiumHashPalette<>(registry, storage.getBits(), this.dummyPaletteResize); + short[] array = this.getOrCreate(strategy.size()); + + storage.compact(this.data.palette(), compactedPalette, array); + + // If the palette didn't change during compaction, do a simple copy of the data array + if (hashPalette != null && hashPalette.getSize() == compactedPalette.getSize() && storage.getBits() == strategy.calculateBitsForSerialization(registry, hashPalette.getSize())) { // paletteSize can de-sync from palette - see https://github.com/CaffeineMC/lithium-fabric/issues/279 + data = this.asOptional(storage.getRaw().clone()); + elements = hashPalette.getElements(); + } else { + int bits = strategy.calculateBitsForSerialization(registry, compactedPalette.getSize()); + if (bits != 0) { + // Re-pack the integer array as the palette has changed size + SimpleBitStorage copy = new SimpleBitStorage(bits, array.length); + for (int i = 0; i < array.length; ++i) { + copy.set(i, array[i]); + } + + // We don't need to clone the data array as we are the sole owner of it + data = this.asOptional(copy.getRaw()); + } + + elements = compactedPalette.getElements(); + } + } } finally { this.release(); } - return var12; + return new PalettedContainerRO.PackedData<>(elements, data); + // Gale end - Lithium - faster chunk serialization } private static void swapPalette(int[] bits, IntUnaryOperator operator) { @@ -368,13 +410,33 @@ public class PalettedContainer implements PaletteResize, PalettedContainer @Override public void count(PalettedContainer.CountConsumer countConsumer) { - if (this.data.palette.getSize() == 1) { - countConsumer.accept(this.data.palette.valueFor(0), this.data.storage.getSize()); - } else { - Int2IntOpenHashMap map = new Int2IntOpenHashMap(); - this.data.storage.getAll(id -> map.addTo(id, 1)); - map.int2IntEntrySet().forEach(idEntry -> countConsumer.accept(this.data.palette.valueFor(idEntry.getIntKey()), idEntry.getIntValue())); + // Gale start - Lithium - faster chunk serialization + int len = this.data.palette().getSize(); + + // Do not allocate huge arrays if we're using a large palette + if (len > 4096) { + // VanillaCopy + if (this.data.palette.getSize() == 1) { + countConsumer.accept(this.data.palette.valueFor(0), this.data.storage.getSize()); + } else { + Int2IntOpenHashMap map = new Int2IntOpenHashMap(); + this.data.storage.getAll(id -> map.addTo(id, 1)); + map.int2IntEntrySet().forEach(idEntry -> countConsumer.accept(this.data.palette.valueFor(idEntry.getIntKey()), idEntry.getIntValue())); + } + } + + short[] counts = new short[len]; + + this.data.storage().getAll(i -> counts[i]++); + + for (int i = 0; i < counts.length; i++) { + T obj = this.data.palette().valueFor(i); + + if (obj != null) { + countConsumer.accept(obj, counts[i]); + } } + // Gale end - Lithium - faster chunk serialization } record Configuration(Palette.Factory factory, int bits) {