From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: jellysquid3 Date: Mon, 10 Jan 2022 15:27:58 -0500 Subject: [PATCH] lithium: chunk.serialization Original code by CaffeineMC, licensed under GNU Lesser General Public License v3.0 You can find the original code on https://github.com/CaffeineMC/lithium-fabric (Yarn mappings) diff --git a/src/main/java/me/jellysquid/mods/lithium/common/world/chunk/LithiumHashPalette.java b/src/main/java/me/jellysquid/mods/lithium/common/world/chunk/LithiumHashPalette.java new file mode 100644 index 0000000000000000000000000000000000000000..16debe176798f316c122e8e7aef2b50ecb9883a6 --- /dev/null +++ b/src/main/java/me/jellysquid/mods/lithium/common/world/chunk/LithiumHashPalette.java @@ -0,0 +1,189 @@ +package me.jellysquid.mods.lithium.common.world.chunk; + +import com.google.common.collect.ImmutableList; +import it.unimi.dsi.fastutil.HashCommon; +import it.unimi.dsi.fastutil.objects.Reference2IntMap; +import it.unimi.dsi.fastutil.objects.Reference2IntOpenHashMap; +import java.util.Arrays; +import java.util.List; +import java.util.function.Predicate; +import net.minecraft.core.IdMap; +import net.minecraft.network.FriendlyByteBuf; +import net.minecraft.world.level.chunk.Palette; +import net.minecraft.world.level.chunk.PaletteResize; + +import static it.unimi.dsi.fastutil.Hash.FAST_LOAD_FACTOR; + +/** + * Generally provides better performance over the vanilla {@link net.minecraft.world.level.chunk.HashMapPalette} when calling + * {@link LithiumHashPalette#idFor(Object)} through using a faster backing map and reducing pointer chasing. + */ +public class LithiumHashPalette implements Palette { + private static final int ABSENT_VALUE = -1; + + private final IdMap idList; + private final PaletteResize resizeHandler; + private final int indexBits; + + private final Reference2IntMap table; + private T[] entries; + private int size = 0; + + public LithiumHashPalette(IdMap idList, PaletteResize resizeHandler, int indexBits, T[] entries, Reference2IntMap table, int size) { + this.idList = idList; + this.resizeHandler = resizeHandler; + this.indexBits = indexBits; + this.entries = entries; + this.table = table; + this.size = size; + } + + public LithiumHashPalette(IdMap idList, int bits, PaletteResize resizeHandler, List list) { + this(idList, bits, resizeHandler); + + for (T t : list) { + this.addEntry(t); + } + } + + @SuppressWarnings("unchecked") + public LithiumHashPalette(IdMap idList, int bits, PaletteResize resizeHandler) { + this.idList = idList; + this.indexBits = bits; + this.resizeHandler = resizeHandler; + + int capacity = 1 << bits; + + this.entries = (T[]) new Object[capacity]; + this.table = new Reference2IntOpenHashMap<>(capacity, FAST_LOAD_FACTOR); + this.table.defaultReturnValue(ABSENT_VALUE); + } + + @Override + public int idFor(T obj) { + int id = this.table.getInt(obj); + + if (id == ABSENT_VALUE) { + id = this.computeEntry(obj); + } + + return id; + } + + @Override + public boolean maybeHas(Predicate predicate) { + for (int i = 0; i < this.size; ++i) { + if (predicate.test(this.entries[i])) { + return true; + } + } + + return false; + } + + private int computeEntry(T obj) { + int id = this.addEntry(obj); + + if (id >= 1 << this.indexBits) { + if (this.resizeHandler == null) { + throw new IllegalStateException("Cannot grow"); + } else { + id = this.resizeHandler.onResize(this.indexBits + 1, obj); + } + } + + return id; + } + + private int addEntry(T obj) { + int nextId = this.size; + + if (nextId >= this.entries.length) { + this.resize(this.size); + } + + this.table.put(obj, nextId); + this.entries[nextId] = obj; + + this.size++; + + return nextId; + } + + private void resize(int neededCapacity) { + this.entries = Arrays.copyOf(this.entries, HashCommon.nextPowerOfTwo(neededCapacity + 1)); + } + + @Override + public T valueFor(int id) { + T[] entries = this.entries; + + if (id >= 0 && id < entries.length) { + return entries[id]; + } + + return null; + } + + @Override + public void read(FriendlyByteBuf buf) { + this.clear(); + + int entryCount = buf.readVarInt(); + + for (int i = 0; i < entryCount; ++i) { + this.addEntry(this.idList.byId(buf.readVarInt())); + } + } + + @Override + public void write(FriendlyByteBuf buf) { + int size = this.size; + buf.writeVarInt(size); + + for (int i = 0; i < size; ++i) { + buf.writeVarInt(this.idList.getId(this.valueFor(i))); + } + } + + @Override + public int getSerializedSize() { + int size = FriendlyByteBuf.getVarIntSize(this.size); + + for (int i = 0; i < this.size; ++i) { + size += FriendlyByteBuf.getVarIntSize(this.idList.getId(this.valueFor(i))); + } + + return size; + } + + @Override + public int getSize() { + return this.size; + } + + @Override + public Palette copy() { + return new LithiumHashPalette<>(this.idList, this.resizeHandler, this.indexBits, this.entries.clone(), new Reference2IntOpenHashMap<>(this.table), this.size); + } + + private void clear() { + Arrays.fill(this.entries, null); + this.table.clear(); + this.size = 0; + } + + public List getElements() { + ImmutableList.Builder builder = new ImmutableList.Builder<>(); + for (T entry : this.entries) { + if (entry != null) { + builder.add(entry); + } + } + return builder.build(); + } + + public static Palette create(int bits, IdMap idList, PaletteResize listener, List list) { + return new LithiumHashPalette<>(idList, bits, listener, list); + } +} \ No newline at end of file diff --git a/src/main/java/net/minecraft/util/BitStorage.java b/src/main/java/net/minecraft/util/BitStorage.java index 106610ccc74b70b557b01c61262d56c4f1147acf..fc986f02290fbe20246022072944980f35dd200c 100644 --- a/src/main/java/net/minecraft/util/BitStorage.java +++ b/src/main/java/net/minecraft/util/BitStorage.java @@ -1,6 +1,7 @@ package net.minecraft.util; import java.util.function.IntConsumer; +import net.minecraft.world.level.chunk.Palette; // JettPack public interface BitStorage { int getAndSet(int index, int value); @@ -31,4 +32,6 @@ public interface BitStorage { } // Paper end + + void compact(Palette srcPalette, Palette dstPalette, short[] out); // JettPack - lithium: chunk.serialization } diff --git a/src/main/java/net/minecraft/util/SimpleBitStorage.java b/src/main/java/net/minecraft/util/SimpleBitStorage.java index 36e33923bf48e56c743ed043bcbc66bc32f0422f..0272dee738e86e066108f5cc3729136335d8197e 100644 --- a/src/main/java/net/minecraft/util/SimpleBitStorage.java +++ b/src/main/java/net/minecraft/util/SimpleBitStorage.java @@ -2,6 +2,7 @@ package net.minecraft.util; import java.util.function.IntConsumer; import javax.annotation.Nullable; +import net.minecraft.world.level.chunk.Palette; // JettPack import org.apache.commons.lang3.Validate; public class SimpleBitStorage implements BitStorage { @@ -201,4 +202,44 @@ public class SimpleBitStorage implements BitStorage { super(message); } } + + // JettPack start - lithium: chunk.serialization + @Override + public void compact(Palette srcPalette, Palette dstPalette, short[] out) { + if (this.size >= Short.MAX_VALUE) { + throw new IllegalStateException("Array too large"); + } + + if (this.size != out.length) { + throw new IllegalStateException("Array size mismatch"); + } + + short[] mappings = new short[(int) (this.mask + 1)]; + + int idx = 0; + + for (long word : this.data) { + long bits = word; + + for (int elementIdx = 0; elementIdx < this.valuesPerLong; ++elementIdx) { + int value = (int) (bits & this.mask); + int remappedId = mappings[value]; + + if (remappedId == 0) { + remappedId = dstPalette.idFor(srcPalette.valueFor(value)) + 1; + mappings[value] = (short) remappedId; + } + + out[idx] = (short) (remappedId - 1); + bits >>= this.bits; + + ++idx; + + if (idx >= this.size) { + return; + } + } + } + } + // JettPack end } diff --git a/src/main/java/net/minecraft/util/ZeroBitStorage.java b/src/main/java/net/minecraft/util/ZeroBitStorage.java index 5d8e9bdf5538b19681f21949368d862fab8a89ad..2224245a985dfe78565dfdb42840d3ed43530525 100644 --- a/src/main/java/net/minecraft/util/ZeroBitStorage.java +++ b/src/main/java/net/minecraft/util/ZeroBitStorage.java @@ -2,6 +2,7 @@ package net.minecraft.util; import java.util.Arrays; import java.util.function.IntConsumer; +import net.minecraft.world.level.chunk.Palette; // JettPack import org.apache.commons.lang3.Validate; public class ZeroBitStorage implements BitStorage { @@ -72,4 +73,6 @@ public class ZeroBitStorage implements BitStorage { public BitStorage copy() { return this; } + + @Override public void compact(Palette srcPalette, Palette dstPalette, short[] out) {} // JettPack } diff --git a/src/main/java/net/minecraft/world/level/chunk/PaletteResize.java b/src/main/java/net/minecraft/world/level/chunk/PaletteResize.java index acae3eb30e0689048937f479dc3070f0688abdad..9c2b79655f2c63a208c7087d5d897db0fb23f697 100644 --- a/src/main/java/net/minecraft/world/level/chunk/PaletteResize.java +++ b/src/main/java/net/minecraft/world/level/chunk/PaletteResize.java @@ -1,5 +1,5 @@ package net.minecraft.world.level.chunk; -interface PaletteResize { +public interface PaletteResize { // JettPack - make public int onResize(int newBits, T object); } diff --git a/src/main/java/net/minecraft/world/level/chunk/PalettedContainer.java b/src/main/java/net/minecraft/world/level/chunk/PalettedContainer.java index 7c770d131d39da6900fdd22df36707d5f43e8cd0..34e98db58044d87c8264c977e16dd17a36182501 100644 --- a/src/main/java/net/minecraft/world/level/chunk/PalettedContainer.java +++ b/src/main/java/net/minecraft/world/level/chunk/PalettedContainer.java @@ -22,8 +22,23 @@ import net.minecraft.util.Mth; import net.minecraft.util.SimpleBitStorage; import net.minecraft.util.ThreadingDetector; import net.minecraft.util.ZeroBitStorage; +import me.jellysquid.mods.lithium.common.world.chunk.LithiumHashPalette; // JettPack public class PalettedContainer implements PaletteResize, PalettedContainerRO { + // JettPack start - lithium: chunk.serialization + private static final ThreadLocal CACHED_ARRAY_4096 = ThreadLocal.withInitial(() -> new short[4096]); + private static final ThreadLocal CACHED_ARRAY_64 = ThreadLocal.withInitial(() -> new short[64]); + private Optional asOptional(long[] data) { + return Optional.of(Arrays.stream(data)); + } + private short[] getOrCreate(int size) { + return switch (size) { + case 64 -> CACHED_ARRAY_64.get(); + case 4096 -> CACHED_ARRAY_4096.get(); + default -> new short[size]; + }; + } + // JettPack end private static final int MIN_PALETTE_BITS = 0; private final PaletteResize dummyPaletteResize = (newSize, added) -> { return 0; @@ -303,30 +318,54 @@ public class PalettedContainer implements PaletteResize, PalettedContainer public synchronized PalettedContainerRO.PackedData pack(IdMap idList, PalettedContainer.Strategy paletteProvider) { // Paper - synchronize this.acquire(); - PalettedContainerRO.PackedData var12; + // JettPack start - lithium: chunk.serialization + Optional data = Optional.empty(); + List elements = null; try { - HashMapPalette hashMapPalette = new HashMapPalette<>(idList, this.data.storage.getBits(), this.dummyPaletteResize); - int i = paletteProvider.size(); - int[] is = new int[i]; - this.data.storage.unpack(is); - swapPalette(is, (id) -> { - return hashMapPalette.idFor(this.data.palette.valueFor(id)); - }); - int j = paletteProvider.calculateBitsForSerialization(idList, hashMapPalette.getSize()); - Optional optional; - if (j != 0) { - SimpleBitStorage simpleBitStorage = new SimpleBitStorage(j, i, is); - optional = Optional.of(Arrays.stream(simpleBitStorage.getRaw())); - } else { - optional = Optional.empty(); + // The palette that will be serialized + LithiumHashPalette hashPalette = null; + + final Palette palette = this.data.palette(); + final BitStorage storage = this.data.storage(); + if (storage instanceof ZeroBitStorage || palette.getSize() == 1) { + // If the palette only contains one entry, don't attempt to repack it. + elements = List.of(palette.valueFor(0)); + } else if (palette instanceof LithiumHashPalette lithiumHashPalette) { + hashPalette = lithiumHashPalette; } - var12 = new PalettedContainerRO.PackedData<>(hashMapPalette.getEntries(), optional); + if (elements == null) { + LithiumHashPalette compactedPalette = new LithiumHashPalette<>(idList, storage.getBits(), this.dummyPaletteResize); + short[] array = this.getOrCreate(paletteProvider.size()); + + storage.compact(this.data.palette(), compactedPalette, array); + + // If the palette didn't change during compaction, do a simple copy of the data array + if (hashPalette != null && hashPalette.getSize() == compactedPalette.getSize() && storage.getBits() == paletteProvider.calculateBitsForSerialization(idList, hashPalette.getSize())) { // paletteSize can de-sync from palette - see https://github.com/CaffeineMC/lithium-fabric/issues/279 + data = this.asOptional(storage.getRaw().clone()); + elements = hashPalette.getElements(); + } else { + int bits = paletteProvider.calculateBitsForSerialization(idList, compactedPalette.getSize()); + if (bits != 0) { + // Re-pack the integer array as the palette has changed size + SimpleBitStorage copy = new SimpleBitStorage(bits, array.length); + for (int i = 0; i < array.length; ++i) { + copy.set(i, array[i]); + } + + // We don't need to clone the data array as we are the sole owner of it + data = this.asOptional(copy.getRaw()); + } + + elements = compactedPalette.getElements(); + } + } } finally { this.release(); } - return var12; + return new PalettedContainerRO.PackedData<>(elements, data); + // JettPack end } private static void swapPalette(int[] is, IntUnaryOperator applier) { @@ -366,17 +405,37 @@ public class PalettedContainer implements PaletteResize, PalettedContainer @Override public void count(PalettedContainer.CountConsumer counter) { - if (this.data.palette.getSize() == 1) { - counter.accept(this.data.palette.valueFor(0), this.data.storage.getSize()); - } else { - Int2IntOpenHashMap int2IntOpenHashMap = new Int2IntOpenHashMap(); - this.data.storage.getAll((key) -> { - int2IntOpenHashMap.addTo(key, 1); - }); - int2IntOpenHashMap.int2IntEntrySet().forEach((entry) -> { - counter.accept(this.data.palette.valueFor(entry.getIntKey()), entry.getIntValue()); - }); + // JettPack start - lithium: chunk.serialization + int len = this.data.palette().getSize(); + + // Do not allocate huge arrays if we're using a large palette + if (len > 4096) { + // VanillaCopy + if (this.data.palette.getSize() == 1) { + counter.accept(this.data.palette.valueFor(0), this.data.storage.getSize()); + } else { + Int2IntOpenHashMap int2IntOpenHashMap = new Int2IntOpenHashMap(); + this.data.storage.getAll((key) -> { + int2IntOpenHashMap.addTo(key, 1); + }); + int2IntOpenHashMap.int2IntEntrySet().forEach((entry) -> { + counter.accept(this.data.palette.valueFor(entry.getIntKey()), entry.getIntValue()); + }); + } + } + + short[] counts = new short[len]; + + this.data.storage().getAll(i -> counts[i]++); + + for (int i = 0; i < counts.length; i++) { + T obj = this.data.palette().valueFor(i); + + if (obj != null) { + counter.accept(obj, counts[i]); + } } + // JettPack end } static record Configuration(Palette.Factory factory, int bits) {