From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: violetc <58360096+s-yh-china@users.noreply.github.com> Date: Tue, 18 Jul 2023 13:14:15 +0800 Subject: [PATCH] Faster chunk serialization This patch is Powered by Gale(https://github.com/GaleMC/Gale) diff --git a/net/minecraft/util/BitStorage.java b/net/minecraft/util/BitStorage.java index 02502d50f0255f5bbcc0ecb965abb48cc1a112da..0abee1cd9d6a5a22d3136e3711de926c3a2d4d73 100644 --- a/net/minecraft/util/BitStorage.java +++ b/net/minecraft/util/BitStorage.java @@ -21,6 +21,8 @@ public interface BitStorage extends ca.spottedleaf.moonrise.patches.block_counti BitStorage copy(); + void compact(net.minecraft.world.level.chunk.Palette srcPalette, net.minecraft.world.level.chunk.Palette dstPalette, short[] out); // Leaves - faster chunk serialization + // Paper start - block counting // provide default impl in case mods implement this... @Override diff --git a/net/minecraft/util/SimpleBitStorage.java b/net/minecraft/util/SimpleBitStorage.java index e6306a68c8652d4c5d22d5ecb1416f5f931f76ee..7b4b3b8a02be3ab5ecafdea1ab74c246daf37d98 100644 --- a/net/minecraft/util/SimpleBitStorage.java +++ b/net/minecraft/util/SimpleBitStorage.java @@ -465,4 +465,44 @@ public class SimpleBitStorage implements BitStorage { super(message); } } + + // Leaves start - faster chunk serialization + @Override + public void compact(net.minecraft.world.level.chunk.Palette srcPalette, net.minecraft.world.level.chunk.Palette dstPalette, short[] out) { + if (this.size >= Short.MAX_VALUE) { + throw new IllegalStateException("Array too large"); + } + + if (this.size != out.length) { + throw new IllegalStateException("Array size mismatch"); + } + + short[] mappings = new short[(int) (this.mask + 1)]; + + int idx = 0; + + for (long word : this.data) { + long bits = word; + + for (int elementIdx = 0; elementIdx < this.valuesPerLong; ++elementIdx) { + int value = (int) (bits & this.mask); + int remappedId = mappings[value]; + + if (remappedId == 0) { + remappedId = dstPalette.idFor(srcPalette.valueFor(value)) + 1; + mappings[value] = (short) remappedId; + } + + out[idx] = (short) (remappedId - 1); + bits >>= this.bits; + + ++idx; + + if (idx >= this.size) { + return; + } + } + } + } + // Leaves end - faster chunk serialization } diff --git a/net/minecraft/util/ZeroBitStorage.java b/net/minecraft/util/ZeroBitStorage.java index 09fd99c9cbd23b5f3c899bfb00c9b89651948ed8..50993ce7519a77c6a9d36cb925125adccda7037f 100644 --- a/net/minecraft/util/ZeroBitStorage.java +++ b/net/minecraft/util/ZeroBitStorage.java @@ -63,6 +63,8 @@ public class ZeroBitStorage implements BitStorage { return this; } + @Override public void compact(net.minecraft.world.level.chunk.Palette srcPalette, net.minecraft.world.level.chunk.Palette dstPalette, short[] out) {} // Leaves - faster chunk serialization + // Paper start - block counting @Override public final it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap moonrise$countEntries() { diff --git a/net/minecraft/world/level/chunk/PaletteResize.java b/net/minecraft/world/level/chunk/PaletteResize.java index c723606fa0be811e580ba47de8c9c575583cc930..c768443c8c6a4b05018bbc70d54b6f41e53e7738 100644 --- a/net/minecraft/world/level/chunk/PaletteResize.java +++ b/net/minecraft/world/level/chunk/PaletteResize.java @@ -1,5 +1,5 @@ package net.minecraft.world.level.chunk; -interface PaletteResize { +public interface PaletteResize { // Leaves - package -> public int onResize(int bits, T objectAdded); } diff --git a/net/minecraft/world/level/chunk/PalettedContainer.java b/net/minecraft/world/level/chunk/PalettedContainer.java index 7da7ce0fd19896593e63edc88b492c02f926bba0..0f89bdb35fced1b9e674959ca510d2da9d4d4c5c 100644 --- a/net/minecraft/world/level/chunk/PalettedContainer.java +++ b/net/minecraft/world/level/chunk/PalettedContainer.java @@ -25,6 +25,24 @@ import net.minecraft.util.ThreadingDetector; import net.minecraft.util.ZeroBitStorage; public class PalettedContainer implements PaletteResize, PalettedContainerRO { + + // Leaves start - faster chunk serialization + private static final ThreadLocal CACHED_ARRAY_4096 = ThreadLocal.withInitial(() -> new short[4096]); + private static final ThreadLocal CACHED_ARRAY_64 = ThreadLocal.withInitial(() -> new short[64]); + + private Optional asOptional(long[] data) { + return Optional.of(Arrays.stream(data)); + } + + private short[] getOrCreate(int size) { + return switch (size) { + case 64 -> CACHED_ARRAY_64.get(); + case 4096 -> CACHED_ARRAY_4096.get(); + default -> new short[size]; + }; + } + // Leaves end - faster chunk serialization + private static final int MIN_PALETTE_BITS = 0; private final PaletteResize dummyPaletteResize = (bits, objectAdded) -> 0; public final IdMap registry; @@ -344,28 +362,76 @@ public class PalettedContainer implements PaletteResize, PalettedContainer public synchronized PalettedContainerRO.PackedData pack(IdMap registry, PalettedContainer.Strategy strategy) { // Paper - synchronize this.acquire(); - PalettedContainerRO.PackedData var12; - try { - HashMapPalette hashMapPalette = new HashMapPalette<>(registry, this.data.storage.getBits(), this.dummyPaletteResize); - int size = strategy.size(); - int[] ints = new int[size]; - this.data.storage.unpack(ints); - swapPalette(ints, id -> hashMapPalette.idFor(this.data.palette.valueFor(id))); - int i = strategy.calculateBitsForSerialization(registry, hashMapPalette.getSize()); - Optional optional; - if (i != 0) { - SimpleBitStorage simpleBitStorage = new SimpleBitStorage(i, size, ints); - optional = Optional.of(Arrays.stream(simpleBitStorage.getRaw())); - } else { - optional = Optional.empty(); - } + // Leaves start - faster chunk serialization + if (!org.leavesmc.leaves.LeavesConfig.performance.fasterChunkSerialization) { + PalettedContainerRO.PackedData var12; + try { + HashMapPalette hashMapPalette = new HashMapPalette<>(registry, this.data.storage.getBits(), this.dummyPaletteResize); + int i = strategy.size(); + int[] is = new int[i]; + this.data.storage.unpack(is); + swapPalette(is, (id) -> { + return hashMapPalette.idFor(this.data.palette.valueFor(id)); + }); + int j = strategy.calculateBitsForSerialization(registry, hashMapPalette.getSize()); + Optional optional; + if (j != 0) { + SimpleBitStorage simpleBitStorage = new SimpleBitStorage(j, i, is); + optional = Optional.of(Arrays.stream(simpleBitStorage.getRaw())); + } else { + optional = Optional.empty(); + } - var12 = new PalettedContainerRO.PackedData<>(hashMapPalette.getEntries(), optional); - } finally { - this.release(); + var12 = new PalettedContainerRO.PackedData<>(hashMapPalette.getEntries(), optional); + } finally { + this.release(); + } + return var12; + } else { + Optional data = Optional.empty(); + List elements = null; + try { + // The palette that will be serialized + org.leavesmc.leaves.lithium.common.world.chunk.LithiumHashPalette hashPalette = null; + + final Palette palette = this.data.palette(); + final BitStorage storage = this.data.storage(); + if (storage instanceof ZeroBitStorage || palette.getSize() == 1) { + // If the palette only contains one entry, don't attempt to repack it. + elements = List.of(palette.valueFor(0)); + } else if (palette instanceof org.leavesmc.leaves.lithium.common.world.chunk.LithiumHashPalette lithiumHashPalette) { + hashPalette = lithiumHashPalette; + } + if (elements == null) { + org.leavesmc.leaves.lithium.common.world.chunk.LithiumHashPalette compactedPalette = new org.leavesmc.leaves.lithium.common.world.chunk.LithiumHashPalette<>(registry, storage.getBits(), this.dummyPaletteResize); + short[] array = this.getOrCreate(strategy.size()); + + storage.compact(this.data.palette(), compactedPalette, array); + + // If the palette didn't change during compaction, do a simple copy of the data array + if (hashPalette != null && hashPalette.getSize() == compactedPalette.getSize() && storage.getBits() == strategy.calculateBitsForSerialization(registry, hashPalette.getSize())) { // paletteSize can de-sync from palette - see https://github.com/CaffeineMC/lithium-fabric/issues/279 + data = this.asOptional(storage.getRaw().clone()); + elements = hashPalette.getElements(); + } else { + int bits = strategy.calculateBitsForSerialization(registry, compactedPalette.getSize()); + if (bits != 0) { + // Re-pack the integer array as the palette has changed size + SimpleBitStorage copy = new SimpleBitStorage(bits, array.length); + for (int i = 0; i < array.length; ++i) { + copy.set(i, array[i]); + } + // We don't need to clone the data array as we are the sole owner of it + data = this.asOptional(copy.getRaw()); + } + } + elements = compactedPalette.getElements(); + } + } finally { + this.release(); + } + return new PalettedContainerRO.PackedData<>(elements, data); } - - return var12; + // Leaves end - faster chunk serialization } private static void swapPalette(int[] bits, IntUnaryOperator operator) { @@ -405,13 +471,47 @@ public class PalettedContainer implements PaletteResize, PalettedContainer @Override public void count(PalettedContainer.CountConsumer countConsumer) { - if (this.data.palette.getSize() == 1) { - countConsumer.accept(this.data.palette.valueFor(0), this.data.storage.getSize()); + // Leaves start - faster chunk serialization + if (!org.leavesmc.leaves.LeavesConfig.performance.fasterChunkSerialization) { + if (this.data.palette.getSize() == 1) { + countConsumer.accept(this.data.palette.valueFor(0), this.data.storage.getSize()); + } else { + Int2IntOpenHashMap int2IntOpenHashMap = new Int2IntOpenHashMap(); + this.data.storage.getAll((key) -> { + int2IntOpenHashMap.addTo(key, 1); + }); + int2IntOpenHashMap.int2IntEntrySet().forEach((entry) -> { + countConsumer.accept(this.data.palette.valueFor(entry.getIntKey()), entry.getIntValue()); + }); + } } else { - Int2IntOpenHashMap map = new Int2IntOpenHashMap(); - this.data.storage.getAll(id -> map.addTo(id, 1)); - map.int2IntEntrySet().forEach(idEntry -> countConsumer.accept(this.data.palette.valueFor(idEntry.getIntKey()), idEntry.getIntValue())); + int len = this.data.palette().getSize(); + + // Do not allocate huge arrays if we're using a large palette + if (len > 4096) { + if (this.data.palette.getSize() == 1) { + countConsumer.accept(this.data.palette.valueFor(0), this.data.storage.getSize()); + } else { + Int2IntOpenHashMap int2IntOpenHashMap = new Int2IntOpenHashMap(); + this.data.storage.getAll((key) -> { + int2IntOpenHashMap.addTo(key, 1); + }); + int2IntOpenHashMap.int2IntEntrySet().forEach((entry) -> { + countConsumer.accept(this.data.palette.valueFor(entry.getIntKey()), entry.getIntValue()); + }); + } + } + + short[] counts = new short[len]; + this.data.storage().getAll(i -> counts[i]++); + for (int i = 0; i < counts.length; i++) { + T obj = this.data.palette().valueFor(i); + if (obj != null) { + countConsumer.accept(obj, counts[i]); + } + } } + // Leaves end - faster chunk serialization } record Configuration(Palette.Factory factory, int bits) {