添加自定义压缩方案

bukkit/loader/build.gradle.kts

@@ -73,5 +73,6 @@ tasks {
         relocate("net.bytebuddy", "net.momirealms.craftengine.libraries.bytebuddy")
         relocate("org.yaml.snakeyaml", "net.momirealms.craftengine.libraries.snakeyaml")
         relocate("org.ahocorasick", "net.momirealms.craftengine.libraries.ahocorasick")
+        relocate("net.jpountz", "net.momirealms.craftengine.libraries.jpountz")
     }
 }

bukkit/loader/src/main/resources/config.yml

@@ -229,6 +229,12 @@ light-system:
   force-update-light: false
 
 chunk-system:
+  # 1 = NONE      |  Compression Speed | Decompress Speed | Compression Ratio | Memory Usage   |
+  # 2 = DEFLATE   |     Medium-Slow           Medium          Moderate              Low        |
+  # 3 = GZIP      |     Medium-Slow           Medium          Moderate              Low        |
+  # 4 = LAZ4      |     Blazing-Fast        Blazing-Fast        Low                 Low        |
+  # 5 = ZSTD      |     Medium-Fast           Fast              High               Medium      |
+  compression-method: 4
   # Disabling this option prevents the plugin from converting custom blocks to vanilla states when chunks are unloaded.
   # While this can improve performance, custom blocks will turn into air if the plugin is uninstalled.
   restore-vanilla-blocks-on-chunk-unload: true

bukkit/loader/src/main/resources/craft-engine.properties

@@ -26,4 +26,5 @@ adventure-text-minimessage=${adventure_bundle_version}
 adventure-text-serializer-gson=${adventure_bundle_version}
 adventure-text-serializer-json=${adventure_bundle_version}
 netty-codec-http=${netty_version}
-ahocorasick=${ahocorasick_version}
+ahocorasick=${ahocorasick_version}
+lz4=${lz4_version}

bukkit/src/main/java/net/momirealms/craftengine/bukkit/world/BukkitWorldManager.java

@@ -212,12 +212,18 @@ public class BukkitWorldManager implements WorldManager, Listener {
                 this.lastVisitedUUID = null;
             }
             this.resetWorldArray();
+
         } finally {
             this.worldMapLock.writeLock().unlock();
         }
         for (Chunk chunk : ((World) world.platformWorld()).getLoadedChunks()) {
             handleChunkUnload(ceWorld, chunk);
         }
+        try {
+            ceWorld.worldDataStorage().close();
+        } catch (IOException e) {
+            CraftEngine.instance().logger().warn("Failed to close world storage", e);
+        }
     }
 
     @Override

core/build.gradle.kts

@@ -79,6 +79,7 @@ tasks {
         relocate("net.kyori", "net.momirealms.craftengine.libraries")
         relocate("org.ahocorasick", "net.momirealms.craftengine.libraries.ahocorasick")
         relocate("net.momirealms.sparrow.nbt", "net.momirealms.craftengine.libraries.nbt")
+        relocate("net.jpountz", "net.momirealms.craftengine.libraries.jpountz") // lz4
     }
 }
 

core/src/main/java/net/momirealms/craftengine/core/plugin/CraftEngine.java

@@ -279,7 +279,8 @@ public abstract class CraftEngine implements Plugin {
                 Dependencies.MINIMESSAGE,
                 Dependencies.TEXT_SERIALIZER_GSON,
                 Dependencies.TEXT_SERIALIZER_JSON,
-                Dependencies.AHO_CORASICK
+                Dependencies.AHO_CORASICK,
+                Dependencies.LZ4
         );
     }
 

core/src/main/java/net/momirealms/craftengine/core/plugin/config/Config.java

@@ -20,6 +20,7 @@ import net.momirealms.craftengine.core.util.AdventureHelper;
 import net.momirealms.craftengine.core.util.Key;
 import net.momirealms.craftengine.core.util.MiscUtils;
 import net.momirealms.craftengine.core.util.ReflectionUtils;
+import net.momirealms.craftengine.core.world.chunk.storage.CompressionMethod;
 
 import java.io.File;
 import java.io.FileInputStream;
@@ -99,6 +100,7 @@ public class Config {
     protected boolean light_system$force_update_light;
     protected boolean light_system$enable;
 
+    protected int chunk_system$compression_method;
     protected boolean chunk_system$restore_vanilla_blocks_on_chunk_unload;
     protected boolean chunk_system$restore_custom_blocks_on_chunk_load;
     protected boolean chunk_system$sync_custom_blocks_on_chunk_load;
@@ -252,6 +254,7 @@ public class Config {
         light_system$enable = config.getBoolean("light-system.enable", true);
 
         // chunk
+        chunk_system$compression_method = config.getInt("chunk-system.compression-method", 4);
         chunk_system$restore_vanilla_blocks_on_chunk_unload = config.getBoolean("chunk-system.restore-vanilla-blocks-on-chunk-unload", true);
         chunk_system$restore_custom_blocks_on_chunk_load = config.getBoolean("chunk-system.restore-custom-blocks-on-chunk-load", true);
         chunk_system$sync_custom_blocks_on_chunk_load = config.getBoolean("chunk-system.sync-custom-blocks-on-chunk-load", false);
@@ -544,27 +547,35 @@ public class Config {
     }
 
     public static boolean filterChat() {
-        return instance().image$illegal_characters_filter$chat;
+        return instance.image$illegal_characters_filter$chat;
     }
 
     public static boolean filterAnvil() {
-        return instance().image$illegal_characters_filter$anvil;
+        return instance.image$illegal_characters_filter$anvil;
     }
 
     public static boolean filterCommand() {
-        return instance().image$illegal_characters_filter$command;
+        return instance.image$illegal_characters_filter$command;
     }
 
     public static boolean filterBook() {
-        return instance().image$illegal_characters_filter$book;
+        return instance.image$illegal_characters_filter$book;
     }
 
     public static boolean filterSign() {
-        return instance().image$illegal_characters_filter$sign;
+        return instance.image$illegal_characters_filter$sign;
     }
 
     public static boolean hideBaseEntity() {
-        return instance().furniture$hide_base_entity;
+        return instance.furniture$hide_base_entity;
+    }
+
+    public static int compressionMethod() {
+        int id = instance.chunk_system$compression_method;
+        if (id <= 0 || id > CompressionMethod.METHOD_COUNT) {
+            id = 4;
+        }
+        return id;
     }
 
     public YamlDocument loadOrCreateYamlData(String fileName) {

core/src/main/java/net/momirealms/craftengine/core/plugin/dependency/Dependencies.java

@@ -214,4 +214,11 @@ public class Dependencies {
             "aho-corasick",
             List.of(Relocation.of("ahocorasick", "org{}ahocorasick"))
     );
+    public static final Dependency LZ4 = new Dependency(
+            "lz4",
+            "org{}lz4",
+            "lz4-java",
+            "lz4-java",
+            List.of(Relocation.of("jpountz", "net{}jpountz"))
+    );
 }

core/src/main/java/net/momirealms/craftengine/core/world/chunk/storage/CompressionMethod.java

@@ -1,24 +1,66 @@
 package net.momirealms.craftengine.core.world.chunk.storage;
 
+import com.github.luben.zstd.ZstdInputStream;
+import com.github.luben.zstd.ZstdOutputStream;
 import it.unimi.dsi.fastutil.io.FastBufferedInputStream;
+import net.jpountz.lz4.LZ4BlockInputStream;
+import net.jpountz.lz4.LZ4BlockOutputStream;
+import net.momirealms.craftengine.core.plugin.CraftEngine;
+import net.momirealms.craftengine.core.plugin.dependency.Dependencies;
 
 import javax.annotation.Nullable;
 import java.io.BufferedOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.lang.reflect.Constructor;
+import java.util.Set;
 import java.util.zip.DeflaterOutputStream;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 import java.util.zip.InflaterInputStream;
 
 public class CompressionMethod {
-    private static final int METHOD_COUNT = 3;
+    public static final int METHOD_COUNT = 5;
     public static final CompressionMethod[] METHODS = new CompressionMethod[METHOD_COUNT +1];
     public static final CompressionMethod NONE = register(new CompressionMethod(1, (stream) -> stream, (stream) -> stream));
     public static final CompressionMethod DEFLATE = register(new CompressionMethod(2, (stream) -> new FastBufferedInputStream(new InflaterInputStream(stream)), (stream) -> new BufferedOutputStream(new DeflaterOutputStream(stream))));
     public static final CompressionMethod GZIP = register(new CompressionMethod(3, (stream) -> new FastBufferedInputStream(new GZIPInputStream(stream)), (stream) -> new BufferedOutputStream(new GZIPOutputStream(stream))));
-//    public static final CompressionMethod LZ4 = register(new CompressionMethod(4, LZ4BlockInputStream::new, LZ4BlockOutputStream::new));
+    public static final CompressionMethod LZ4 = register(new CompressionMethod(4, LZ4BlockInputStream::new, LZ4BlockOutputStream::new));
+    public static final CompressionMethod ZSTD;
+
+    static {
+        ClassLoader classLoader = CraftEngine.instance().dependencyManager().obtainClassLoaderWith(Set.of(Dependencies.ZSTD));
+        try {
+            Class<?> inputStreamClass = classLoader.loadClass("com.github.luben.zstd.ZstdInputStream");
+            Constructor<?> inputStreamConstructor = inputStreamClass.getConstructor(InputStream.class);
+            Class<?> outputStreamClass = classLoader.loadClass("com.github.luben.zstd.ZstdOutputStream");
+            Constructor<?> outputStreamConstructor = outputStreamClass.getConstructor(OutputStream.class);
+            ZSTD = register(
+                    new CompressionMethod(
+                            5,
+                            rawStream -> {
+                                try {
+                                    return (InputStream) inputStreamConstructor.newInstance(rawStream);
+                                } catch (Exception e) {
+                                    CraftEngine.instance().logger().warn("Could not instantiate ZstdInputStream", e);
+                                    return rawStream;
+                                }
+                            },
+                            rawStream -> {
+                                try {
+                                    return (OutputStream) outputStreamConstructor.newInstance(rawStream);
+                                } catch (Exception e) {
+                                    CraftEngine.instance().logger().warn("Could not instantiate ZstdOutputStream", e);
+                                    return rawStream;
+                                }
+                            }
+                    )
+            );
+        } catch (ReflectiveOperationException e) {
+            throw new RuntimeException(e);
+        }
+    }
 
     private final int id;
     private final StreamWrapper<InputStream> inputWrapper;

core/src/main/java/net/momirealms/craftengine/core/world/chunk/storage/DefaultRegionFileStorage.java

@@ -2,6 +2,7 @@ package net.momirealms.craftengine.core.world.chunk.storage;
 
 import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
 import it.unimi.dsi.fastutil.longs.LongLinkedOpenHashSet;
+import net.momirealms.craftengine.core.plugin.config.Config;
 import net.momirealms.craftengine.core.util.ExceptionCollector;
 import net.momirealms.craftengine.core.util.FileUtils;
 import net.momirealms.craftengine.core.world.ChunkPos;
@@ -14,6 +15,7 @@ import java.io.DataOutputStream;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.stream.Stream;
 
 public class DefaultRegionFileStorage implements WorldDataStorage {
     private static final int FORMAT_VERSION = 1;
@@ -21,13 +23,22 @@ public class DefaultRegionFileStorage implements WorldDataStorage {
 
     public static final String REGION_FILE_SUFFIX = ".mca";
     public static final String REGION_FILE_PREFIX = "r.";
+    public static final int MAX_NON_EXISTING_CACHE = 1024 * 64;
 
     public final Long2ObjectLinkedOpenHashMap<RegionFile> regionCache = new Long2ObjectLinkedOpenHashMap<>();
     private final LongLinkedOpenHashSet nonExistingRegionFiles = new LongLinkedOpenHashSet();
-    static final int MAX_NON_EXISTING_CACHE = 1024 * 64;
 
     public DefaultRegionFileStorage(Path directory) {
         this.folder = directory;
+        Path metaDataFile = this.folder.getParent().resolve("craftengine.dat");
+    }
+
+    private Path[] regionFiles() throws IOException {
+        try (Stream<Path> paths = Files.walk(this.folder)) {
+            return paths
+                    .filter(path -> path.toString().endsWith(REGION_FILE_SUFFIX))
+                    .toArray(Path[]::new);
+        }
     }
 
     @SuppressWarnings("BooleanMethodIsAlwaysInverted")
@@ -88,7 +99,7 @@ public class DefaultRegionFileStorage implements WorldDataStorage {
                 this.createRegionFile(chunkPosLongKey);
             }
             FileUtils.createDirectoriesSafe(this.folder);
-            RegionFile newRegionFile = new RegionFile(path, this.folder);
+            RegionFile newRegionFile = new RegionFile(path, this.folder, CompressionMethod.fromId(Config.compressionMethod()));
 
             this.regionCache.putAndMoveToFirst(chunkPosLongKey, newRegionFile);
             if (lock) {

core/src/main/java/net/momirealms/craftengine/core/world/chunk/storage/RegionFile.java

@@ -42,10 +42,6 @@ public class RegionFile implements AutoCloseable {
     public final ReentrantLock fileLock = new ReentrantLock(true);
     public final Path regionFile;
 
-    public RegionFile(Path fileChannel, Path directory) throws IOException {
-        this(fileChannel, directory, CompressionMethod.GZIP);
-    }
-
     public RegionFile(Path path, Path directory, CompressionMethod compressionMethod) throws IOException {
         this.header = ByteBuffer.allocateDirect(8192);
         this.regionFile = path;
@@ -400,10 +396,12 @@ public class RegionFile implements AutoCloseable {
 
         public ChunkBuffer(ChunkPos pos) {
             super(8096);
+            // chunk size 4 bytes
             super.write(0);
             super.write(0);
             super.write(0);
             super.write(0);
+            // compression method
             super.write(RegionFile.this.compression.getId());
             this.pos = pos;
         }