From 9922348a25bd5b7763f8418bc8e8558d6afd01f0 Mon Sep 17 00:00:00 2001 From: NONPLAYT <76615486+NONPLAYT@users.noreply.github.com> Date: Sat, 18 Jan 2025 03:52:05 +0300 Subject: [PATCH] add base linear region (we have some bugs with it) --- README.md | 4 +- build-data/divinemc.at | 5 + .../0055-Implement-Linear-region-format.patch | 858 ------------------ divinemc-server/build.gradle.kts.patch | 10 + .../0013-Implement-Linear-region-format.patch | 362 ++++++++ .../divinemc/configuration/DivineConfig.java | 12 + .../configuration/DivineWorldConfig.java | 20 + .../divinemc/region/AbstractRegionFile.java | 27 + .../region/AbstractRegionFileFactory.java | 25 + .../divinemc/region/LinearRegionFile.java | 298 ++++++ .../divinemc/region/RegionFileFormat.java | 16 + 11 files changed, 777 insertions(+), 860 deletions(-) delete mode 100644 divinemc-archived-patches/work/server/0055-Implement-Linear-region-format.patch create mode 100644 divinemc-server/minecraft-patches/features/0013-Implement-Linear-region-format.patch create mode 100644 divinemc-server/src/main/java/space/bxteam/divinemc/region/AbstractRegionFile.java create mode 100644 divinemc-server/src/main/java/space/bxteam/divinemc/region/AbstractRegionFileFactory.java create mode 100644 divinemc-server/src/main/java/space/bxteam/divinemc/region/LinearRegionFile.java create mode 100644 divinemc-server/src/main/java/space/bxteam/divinemc/region/RegionFileFormat.java diff --git a/README.md b/README.md index acd6dbf..9f1ab74 100644 --- a/README.md +++ b/README.md @@ -77,11 +77,11 @@ First, clone this repository (do not download it) and the run the following comm After that, project is ready to use and editing it. ### Creating a patch -Patches are effectively just commits in either `paper-api`, `paper-server`, `purpur-api`, `purpur-server` or `divinmc-server`. If you want to learn how to work with patch system, you can read our [contributing documentation](https://docs.bx-team.space/documentation/divinemc/development/contributing). +Patches are effectively just commits in either `paper-api`, `paper-server`, `purpur-api`, `purpur-server` or `divinemc-server`. If you want to learn how to work with patch system, you can read our [contributing documentation](https://docs.bx-team.space/documentation/divinemc/development/contributing). ### Compiling Use the command `./gradlew build` to build the API and server. Compiled JARs will be placed under `divinemc-api/build/libs` and `divinemc-server/build/libs`. **These JARs are not used to start a server**. -To compile a server-ready paperclip jar, run `./gradlew createMojmapPaperclipJar`. The compiled paperclip jar will be in `divinemc-server/build/libs` in the main root. +To compile a server-ready paperclip jar, run `./gradlew createMojmapPaperclipJar`. The compiled paperclip jar will be put in `divinemc-server/build/libs`. ###### We don't steal logo from YatopiaMC! [List of all forks](https://gist.github.com/NONPLAYT/48742353af8ae36bcef5d1c36de9730a) diff --git a/build-data/divinemc.at b/build-data/divinemc.at index ae16fb1..7239a5d 100644 --- a/build-data/divinemc.at +++ b/build-data/divinemc.at @@ -1,4 +1,9 @@ # This file is auto generated, any changes may be overridden! # See CONTRIBUTING.md on how to add access transformers. public net.minecraft.world.level.block.state.pattern.BlockPattern matches(Lnet/minecraft/core/BlockPos;Lnet/minecraft/core/Direction;Lnet/minecraft/core/Direction;Lcom/google/common/cache/LoadingCache;)Lnet/minecraft/world/level/block/state/pattern/BlockPattern$BlockPatternMatch; +public net.minecraft.world.level.chunk.storage.RegionFile getOversizedData(II)Lnet/minecraft/nbt/CompoundTag; +public net.minecraft.world.level.chunk.storage.RegionFile isOversized(II)Z +public net.minecraft.world.level.chunk.storage.RegionFile recalculateHeader()Z +public net.minecraft.world.level.chunk.storage.RegionFile setOversized(IIZ)V +public net.minecraft.world.level.chunk.storage.RegionFile write(Lnet/minecraft/world/level/ChunkPos;Ljava/nio/ByteBuffer;)V public net.minecraft.world.level.pathfinder.SwimNodeEvaluator allowBreaching diff --git a/divinemc-archived-patches/work/server/0055-Implement-Linear-region-format.patch b/divinemc-archived-patches/work/server/0055-Implement-Linear-region-format.patch deleted file mode 100644 index 5e8938f..0000000 --- a/divinemc-archived-patches/work/server/0055-Implement-Linear-region-format.patch +++ /dev/null @@ -1,858 +0,0 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: NONPLAYT <76615486+NONPLAYT@users.noreply.github.com> -Date: Wed, 10 Jul 2024 02:40:08 +0300 -Subject: [PATCH] Implement Linear region format - -Status: may be replaced with SlimeWorldManager soon - -diff --git a/build.gradle.kts b/build.gradle.kts -index 3e2a092c797ec7918f5c4b838f28b0778c70531c..bb44922202e3cdb705a4773ea7c9ec807b5f3de2 100644 ---- a/build.gradle.kts -+++ b/build.gradle.kts -@@ -30,6 +30,10 @@ dependencies { - alsoShade(log4jPlugins.output) - implementation("io.netty:netty-codec-haproxy:4.1.97.Final") // Paper - Add support for proxy protocol - // Paper end -+ // DivineMC start -+ implementation("com.github.luben:zstd-jni:1.5.6-3") -+ implementation("org.lz4:lz4-java:1.8.0") -+ // DivineMC end - implementation("org.apache.logging.log4j:log4j-iostreams:2.22.1") // Paper - remove exclusion - implementation("org.ow2.asm:asm-commons:9.7.1") - implementation("org.spongepowered:configurate-yaml:4.2.0-SNAPSHOT") // Paper - config files -diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java -index 73df26b27146bbad2106d57b22dd3c792ed3dd1d..a4d16996fae07f943ee078ce3d2e7b22747fc2d1 100644 ---- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java -+++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java -@@ -7,8 +7,8 @@ public interface ChunkSystemRegionFileStorage { - - public boolean moonrise$doesRegionFileNotExistNoIO(final int chunkX, final int chunkZ); - -- public RegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ); -+ public space.bxteam.divinemc.region.AbstractRegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ); // DivineMC - -- public RegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException; -+ public space.bxteam.divinemc.region.AbstractRegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException; // DivineMC - - } -diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/RegionFileIOThread.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/RegionFileIOThread.java -index 3218cbf84f54daf06e84442d5eb1a36d8da6b215..0db1f8f64a261780e6692755669fa573a2c2b199 100644 ---- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/RegionFileIOThread.java -+++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/RegionFileIOThread.java -@@ -1043,9 +1043,9 @@ public final class RegionFileIOThread extends PrioritisedQueueExecutorThread { - return ((ChunkSystemRegionFileStorage)(Object)this.getCache()).moonrise$doesRegionFileNotExistNoIO(chunkX, chunkZ); - } - -- public T computeForRegionFile(final int chunkX, final int chunkZ, final boolean existingOnly, final Function function) { -+ public T computeForRegionFile(final int chunkX, final int chunkZ, final boolean existingOnly, final Function function) { // DivineMC - final RegionFileStorage cache = this.getCache(); -- final RegionFile regionFile; -+ final space.bxteam.divinemc.region.AbstractRegionFile regionFile; // DivineMC - synchronized (cache) { - try { - if (existingOnly) { -@@ -1061,9 +1061,9 @@ public final class RegionFileIOThread extends PrioritisedQueueExecutorThread { - } - } - -- public T computeForRegionFileIfLoaded(final int chunkX, final int chunkZ, final Function function) { -+ public T computeForRegionFileIfLoaded(final int chunkX, final int chunkZ, final Function function) { // DivineMC - final RegionFileStorage cache = this.getCache(); -- final RegionFile regionFile; -+ final space.bxteam.divinemc.region.AbstractRegionFile regionFile; - - synchronized (cache) { - regionFile = ((ChunkSystemRegionFileStorage)(Object)cache).moonrise$getRegionFileIfLoaded(chunkX, chunkZ); -diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java -index 8658296b919fd6fa28e64a64186060d3704271db..52bdfd853cf9d42a6c58f0616faf031c8e91fed8 100644 ---- a/src/main/java/net/minecraft/server/MinecraftServer.java -+++ b/src/main/java/net/minecraft/server/MinecraftServer.java -@@ -981,10 +981,10 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop> progressMap = Reference2FloatMaps.synchronize(new Reference2FloatOpenHashMap()); - volatile Component status = Component.translatable("optimizeWorld.stage.counting"); -- static final Pattern REGEX = Pattern.compile("^r\\.(-?[0-9]+)\\.(-?[0-9]+)\\.mca$"); -+ static final Pattern REGEX = Pattern.compile("^r\\.(-?[0-9]+)\\.(-?[0-9]+)\\.(linear | mca)$"); // DivineMC - final DimensionDataStorage overworldDataStorage; - - public WorldUpgrader(LevelStorageSource.LevelStorageAccess session, DataFixer dataFixer, RegistryAccess dynamicRegistryManager, boolean eraseCache, boolean recreateRegionFiles) { -@@ -400,7 +400,7 @@ public class WorldUpgrader { - - private static List getAllChunkPositions(RegionStorageInfo key, Path regionDirectory) { - File[] afile = regionDirectory.toFile().listFiles((file, s) -> { -- return s.endsWith(".mca"); -+ return s.endsWith(".linear") || s.endsWith(".mca"); // DivineMC - }); - - if (afile == null) { -@@ -420,7 +420,7 @@ public class WorldUpgrader { - List list1 = Lists.newArrayList(); - - try { -- RegionFile regionfile = new RegionFile(key, file.toPath(), regionDirectory, true); -+ space.bxteam.divinemc.region.AbstractRegionFile regionfile = space.bxteam.divinemc.region.AbstractRegionFileFactory.getAbstractRegionFile(key, file.toPath(), regionDirectory, true); // DivineMC - - try { - for (int i1 = 0; i1 < 32; ++i1) { -@@ -483,7 +483,7 @@ public class WorldUpgrader { - - protected abstract boolean tryProcessOnePosition(T storage, ChunkPos chunkPos, ResourceKey worldKey); - -- private void onFileFinished(RegionFile regionFile) { -+ private void onFileFinished(space.bxteam.divinemc.region.AbstractRegionFile regionFile) { // DivineMC - if (WorldUpgrader.this.recreateRegionFiles) { - if (this.previousWriteFuture != null) { - this.previousWriteFuture.join(); -@@ -508,7 +508,7 @@ public class WorldUpgrader { - } - } - -- static record FileToUpgrade(RegionFile file, List chunksToUpgrade) { -+ static record FileToUpgrade(space.bxteam.divinemc.region.AbstractRegionFile file, List chunksToUpgrade) { // DivineMC - - } - -diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java -index 1e0439cf3f4008fa430acb90b45f5bc4cdd6d7f2..05b356177fc0f65de51f03f7ef1a2f03d85e92ff 100644 ---- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java -+++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java -@@ -28,7 +28,7 @@ import net.minecraft.nbt.NbtIo; // Paper - import net.minecraft.world.level.ChunkPos; - import org.slf4j.Logger; - --public class RegionFile implements AutoCloseable { -+public class RegionFile implements AutoCloseable, space.bxteam.divinemc.region.AbstractRegionFile { // DivineMC - - private static final Logger LOGGER = LogUtils.getLogger(); - private static final int SECTOR_BYTES = 4096; -@@ -129,7 +129,7 @@ public class RegionFile implements AutoCloseable { - } - - // note: only call for CHUNK regionfiles -- boolean recalculateHeader() throws IOException { -+ public boolean recalculateHeader() throws IOException { // DivineMC - make public - if (!this.canRecalcHeader) { - return false; - } -@@ -928,10 +928,10 @@ public class RegionFile implements AutoCloseable { - private static int getChunkIndex(int x, int z) { - return (x & 31) + (z & 31) * 32; - } -- synchronized boolean isOversized(int x, int z) { -+ public synchronized boolean isOversized(int x, int z) { // DivineMC - return this.oversized[getChunkIndex(x, z)] == 1; - } -- synchronized void setOversized(int x, int z, boolean oversized) throws IOException { -+ public synchronized void setOversized(int x, int z, boolean oversized) throws IOException { // DivineMC - final int offset = getChunkIndex(x, z); - boolean previous = this.oversized[offset] == 1; - this.oversized[offset] = (byte) (oversized ? 1 : 0); -@@ -970,7 +970,7 @@ public class RegionFile implements AutoCloseable { - return this.path.getParent().resolve(this.path.getFileName().toString().replaceAll("\\.mca$", "") + "_oversized_" + x + "_" + z + ".nbt"); - } - -- synchronized CompoundTag getOversizedData(int x, int z) throws IOException { -+ public synchronized CompoundTag getOversizedData(int x, int z) throws IOException { // DivineMC - Path file = getOversizedFile(x, z); - try (DataInputStream out = new DataInputStream(new java.io.BufferedInputStream(new InflaterInputStream(Files.newInputStream(file))))) { - return NbtIo.read((java.io.DataInput) out); -diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java -index e623bb14c44a734607e7c6365620d59a3db9f1da..f64715e4fc875dcd714e3d5e9b4d53f32b8ea2e7 100644 ---- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java -+++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java -@@ -21,7 +21,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise - - public static final String ANVIL_EXTENSION = ".mca"; - private static final int MAX_CACHE_SIZE = 256; -- public final Long2ObjectLinkedOpenHashMap regionCache = new Long2ObjectLinkedOpenHashMap(); -+ public final Long2ObjectLinkedOpenHashMap regionCache = new Long2ObjectLinkedOpenHashMap(); // DivineMC - private final RegionStorageInfo info; - private final Path folder; - private final boolean sync; -@@ -30,7 +30,9 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise - private static final int REGION_SHIFT = 5; - private static final int MAX_NON_EXISTING_CACHE = 1024 * 64; - private final it.unimi.dsi.fastutil.longs.LongLinkedOpenHashSet nonExistingRegionFiles = new it.unimi.dsi.fastutil.longs.LongLinkedOpenHashSet(MAX_NON_EXISTING_CACHE+1); -- private static String getRegionFileName(final int chunkX, final int chunkZ) { -+ private static String getRegionFileName(final RegionStorageInfo info, final int chunkX, final int chunkZ) { -+ if (info.regionFormat().equals(space.bxteam.divinemc.region.RegionFileFormat.LINEAR)) -+ return "r." + (chunkX >> REGION_SHIFT) + "." + (chunkZ >> REGION_SHIFT) + ".linear"; // DivineMC - linear - return "r." + (chunkX >> REGION_SHIFT) + "." + (chunkZ >> REGION_SHIFT) + ".mca"; - } - -@@ -66,15 +68,15 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise - } - - @Override -- public synchronized final RegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ) { -+ public synchronized final space.bxteam.divinemc.region.AbstractRegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ) { // DivineMC - return this.regionCache.getAndMoveToFirst(ChunkPos.asLong(chunkX >> REGION_SHIFT, chunkZ >> REGION_SHIFT)); - } - - @Override -- public synchronized final RegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException { -+ public synchronized final space.bxteam.divinemc.region.AbstractRegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException { // DivineMC - final long key = ChunkPos.asLong(chunkX >> REGION_SHIFT, chunkZ >> REGION_SHIFT); - -- RegionFile ret = this.regionCache.getAndMoveToFirst(key); -+ space.bxteam.divinemc.region.AbstractRegionFile ret = this.regionCache.getAndMoveToFirst(key); // DivineMC - if (ret != null) { - return ret; - } -@@ -87,7 +89,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise - this.regionCache.removeLast().close(); - } - -- final Path regionPath = this.folder.resolve(getRegionFileName(chunkX, chunkZ)); -+ final Path regionPath = this.folder.resolve(getRegionFileName(this.info, chunkX, chunkZ)); // DivineMC - - if (!java.nio.file.Files.exists(regionPath)) { - this.markNonExisting(key); -@@ -98,7 +100,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise - - FileUtil.createDirectoriesSafe(this.folder); - -- ret = new RegionFile(this.info, regionPath, this.folder, this.sync); -+ ret = space.bxteam.divinemc.region.AbstractRegionFileFactory.getAbstractRegionFile(this.info, regionPath, this.folder, this.sync); // DivineMC - - this.regionCache.putAndMoveToFirst(key, ret); - -@@ -143,7 +145,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise - this.isChunkData = isChunkDataFolder(this.folder); // Paper - recalculate region file headers - } - -- public RegionFile getRegionFile(ChunkPos chunkcoordintpair, boolean existingOnly) throws IOException { // CraftBukkit // Paper - public -+ public space.bxteam.divinemc.region.AbstractRegionFile getRegionFile(ChunkPos chunkcoordintpair, boolean existingOnly) throws IOException { // CraftBukkit // Paper - public // DivineMC - // Paper start - rewrite chunk system - if (existingOnly) { - return this.moonrise$getRegionFileIfExists(chunkcoordintpair.x, chunkcoordintpair.z); -@@ -151,7 +153,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise - synchronized (this) { - final long key = ChunkPos.asLong(chunkcoordintpair.x >> REGION_SHIFT, chunkcoordintpair.z >> REGION_SHIFT); - -- RegionFile ret = this.regionCache.getAndMoveToFirst(key); -+ space.bxteam.divinemc.region.AbstractRegionFile ret = this.regionCache.getAndMoveToFirst(key); // DivineMC - if (ret != null) { - return ret; - } -@@ -160,13 +162,13 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise - this.regionCache.removeLast().close(); - } - -- final Path regionPath = this.folder.resolve(getRegionFileName(chunkcoordintpair.x, chunkcoordintpair.z)); -+ final Path regionPath = this.folder.resolve(getRegionFileName(this.info, chunkcoordintpair.x, chunkcoordintpair.z)); // DivineMC - - this.createRegionFile(key); - - FileUtil.createDirectoriesSafe(this.folder); - -- ret = new RegionFile(this.info, regionPath, this.folder, this.sync); -+ ret = space.bxteam.divinemc.region.AbstractRegionFileFactory.getAbstractRegionFile(this.info, regionPath, this.folder, this.sync); // DivineMC - - this.regionCache.putAndMoveToFirst(key, ret); - -@@ -180,7 +182,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise - org.apache.logging.log4j.LogManager.getLogger().fatal(msg + " (" + file.toString().replaceAll(".+[\\\\/]", "") + " - " + x + "," + z + ") Go clean it up to remove this message. /minecraft:tp " + (x<<4)+" 128 "+(z<<4) + " - DO NOT REPORT THIS TO DIVINEMC - You may ask for help on Discord, but do not file an issue. These error messages can not be removed."); // DivineMC - } - -- private static CompoundTag readOversizedChunk(RegionFile regionfile, ChunkPos chunkCoordinate) throws IOException { -+ private static CompoundTag readOversizedChunk(space.bxteam.divinemc.region.AbstractRegionFile regionfile, ChunkPos chunkCoordinate) throws IOException { // DivineMC - synchronized (regionfile) { - try (DataInputStream datainputstream = regionfile.getChunkDataInputStream(chunkCoordinate)) { - CompoundTag oversizedData = regionfile.getOversizedData(chunkCoordinate.x, chunkCoordinate.z); -@@ -215,7 +217,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise - @Nullable - public CompoundTag read(ChunkPos pos) throws IOException { - // CraftBukkit start - SPIGOT-5680: There's no good reason to preemptively create files on read, save that for writing -- RegionFile regionfile = this.getRegionFile(pos, true); -+ space.bxteam.divinemc.region.AbstractRegionFile regionfile = this.getRegionFile(pos, true); // DivineMC - if (regionfile == null) { - return null; - } -@@ -279,7 +281,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise - - public void scanChunk(ChunkPos chunkPos, StreamTagVisitor scanner) throws IOException { - // CraftBukkit start - SPIGOT-5680: There's no good reason to preemptively create files on read, save that for writing -- RegionFile regionfile = this.getRegionFile(chunkPos, true); -+ space.bxteam.divinemc.region.AbstractRegionFile regionfile = this.getRegionFile(chunkPos, true); // DivineMC - if (regionfile == null) { - return; - } -@@ -309,7 +311,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise - } - - public void write(ChunkPos pos, @Nullable CompoundTag nbt) throws IOException { // Paper - public -- RegionFile regionfile = this.getRegionFile(pos, nbt == null); // CraftBukkit // Paper - rewrite chunk system -+ space.bxteam.divinemc.region.AbstractRegionFile regionfile = this.getRegionFile(pos, nbt == null); // CraftBukkit // Paper - rewrite chunk system // DivineMC - // Paper start - rewrite chunk system - if (regionfile == null) { - // if the RegionFile doesn't exist, no point in deleting from it -@@ -368,7 +370,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise - // Paper start - rewrite chunk system - synchronized (this) { - final ExceptionCollector exceptionCollector = new ExceptionCollector<>(); -- for (final RegionFile regionFile : this.regionCache.values()) { -+ for (final space.bxteam.divinemc.region.AbstractRegionFile regionFile : this.regionCache.values()) { // DivineMC - try { - regionFile.close(); - } catch (final IOException ex) { -@@ -385,7 +387,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise - // Paper start - rewrite chunk system - synchronized (this) { - final ExceptionCollector exceptionCollector = new ExceptionCollector<>(); -- for (final RegionFile regionFile : this.regionCache.values()) { -+ for (final space.bxteam.divinemc.region.AbstractRegionFile regionFile : this.regionCache.values()) { // DivineMC - try { - regionFile.flush(); - } catch (final IOException ex) { -diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionStorageInfo.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionStorageInfo.java -index 6111631c6673948b266286894603cc5e30451b02..8bfc0fe2b8389359a682458fde5ef1b0670f7a2d 100644 ---- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionStorageInfo.java -+++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionStorageInfo.java -@@ -7,4 +7,20 @@ public record RegionStorageInfo(String level, ResourceKey dimension, Stri - public RegionStorageInfo withTypeSuffix(String suffix) { - return new RegionStorageInfo(this.level, this.dimension, this.type + suffix); - } -+ -+ // DivineMC start -+ public space.bxteam.divinemc.region.RegionFileFormat regionFormat() { -+ return ((org.bukkit.craftbukkit.CraftWorld) org.bukkit.Bukkit.getWorld(level)) -+ .getHandle() -+ .divinemcConfig -+ .regionFormat; -+ } -+ -+ public int linearCompressionLevel() { -+ return ((org.bukkit.craftbukkit.CraftWorld) org.bukkit.Bukkit.getWorld(level)) -+ .getHandle() -+ .divinemcConfig -+ .linearCompressionLevel; -+ } -+ // DivineMC end - } -diff --git a/src/main/java/space/bxteam/divinemc/configuration/DivineConfig.java b/src/main/java/space/bxteam/divinemc/configuration/DivineConfig.java -index acf4a5287ba8d1ec8b4157426aa5794211654b7b..e8cbdcc5c41acfc18fa040997421fb712de801cb 100644 ---- a/src/main/java/space/bxteam/divinemc/configuration/DivineConfig.java -+++ b/src/main/java/space/bxteam/divinemc/configuration/DivineConfig.java -@@ -194,4 +194,16 @@ public class DivineConfig { - private static void chatMessageSignatures() { - noChatSign = getBoolean("settings.no-chat-sign", noChatSign); - } -+ -+ public static int linearFlushFrequency = 10; -+ public static int linearFlushThreads = 1; -+ private static void linearSettings() { -+ linearFlushFrequency = getInt("settings.region-format.linear.flush-frequency", linearFlushFrequency); -+ linearFlushThreads = getInt("settings.region-format.linear.flush-max-threads", linearFlushThreads); -+ -+ if (linearFlushThreads < 0) -+ linearFlushThreads = Math.max(Runtime.getRuntime().availableProcessors() + linearFlushThreads, 1); -+ else -+ linearFlushThreads = Math.max(linearFlushThreads, 1); -+ } - } -diff --git a/src/main/java/space/bxteam/divinemc/configuration/DivineWorldConfig.java b/src/main/java/space/bxteam/divinemc/configuration/DivineWorldConfig.java -index d94c51ea18d299dd52b9a8521a9cdc0d95b79356..f216f728dd5ca6e7b67f4f1384ce73faf9814384 100644 ---- a/src/main/java/space/bxteam/divinemc/configuration/DivineWorldConfig.java -+++ b/src/main/java/space/bxteam/divinemc/configuration/DivineWorldConfig.java -@@ -3,10 +3,12 @@ package space.bxteam.divinemc.configuration; - import org.apache.commons.lang.BooleanUtils; - import org.bukkit.World; - import org.bukkit.configuration.ConfigurationSection; -+import space.bxteam.divinemc.region.RegionFileFormat; - - import java.util.List; - import java.util.Map; - import java.util.function.Predicate; -+import java.util.logging.Level; - - import static space.bxteam.divinemc.configuration.DivineConfig.log; - -@@ -106,4 +108,22 @@ public class DivineWorldConfig { - private void suppressErrorsFromDirtyAttributes() { - suppressErrorsFromDirtyAttributes = getBoolean("suppress-errors-from-dirty-attributes", suppressErrorsFromDirtyAttributes); - } -+ -+ public RegionFileFormat regionFormat = RegionFileFormat.ANVIL; -+ public int linearCompressionLevel = 1; -+ private void regionFormatSettings() { -+ regionFormat = RegionFileFormat.fromString(getString("region-format.format", regionFormat.name())); -+ if (regionFormat.equals(RegionFileFormat.INVALID)) { -+ log(Level.SEVERE, "Unknown region format in linear.yml: " + regionFormat); -+ log(Level.SEVERE, "Falling back to ANVIL region file format."); -+ regionFormat = RegionFileFormat.ANVIL; -+ } -+ -+ linearCompressionLevel = getInt("region-format.linear.compression-level", linearCompressionLevel); -+ if (linearCompressionLevel > 23 || linearCompressionLevel < 1) { -+ log(Level.SEVERE, "Linear region compression level should be between 1 and 22 in linear.yml: " + linearCompressionLevel); -+ log(Level.SEVERE, "Falling back to compression level 1."); -+ linearCompressionLevel = 1; -+ } -+ } - } -diff --git a/src/main/java/space/bxteam/divinemc/region/AbstractRegionFile.java b/src/main/java/space/bxteam/divinemc/region/AbstractRegionFile.java -new file mode 100644 -index 0000000000000000000000000000000000000000..8bc4681e38ab29dae8173492d3430986d8baa164 ---- /dev/null -+++ b/src/main/java/space/bxteam/divinemc/region/AbstractRegionFile.java -@@ -0,0 +1,26 @@ -+package space.bxteam.divinemc.region; -+ -+import java.io.DataInputStream; -+import java.io.DataOutputStream; -+import java.io.IOException; -+import java.nio.file.Path; -+import net.minecraft.nbt.CompoundTag; -+import net.minecraft.world.level.ChunkPos; -+ -+public interface AbstractRegionFile { -+ -+ Path getPath(); -+ DataInputStream getChunkDataInputStream(ChunkPos pos) throws IOException; -+ DataOutputStream getChunkDataOutputStream(ChunkPos pos) throws IOException; -+ CompoundTag getOversizedData(int x, int z) throws IOException; -+ -+ boolean hasChunk(ChunkPos pos); -+ boolean doesChunkExist(ChunkPos pos); -+ boolean isOversized(int x, int z); -+ boolean recalculateHeader() throws IOException; -+ -+ void flush() throws IOException; -+ void close() throws IOException; -+ void clear(ChunkPos pos) throws IOException; -+ void setOversized(int x, int z, boolean oversized) throws IOException; -+} -diff --git a/src/main/java/space/bxteam/divinemc/region/AbstractRegionFileFactory.java b/src/main/java/space/bxteam/divinemc/region/AbstractRegionFileFactory.java -new file mode 100644 -index 0000000000000000000000000000000000000000..eb019b4863c2f812902318bdc8f3ab34ddb7aac5 ---- /dev/null -+++ b/src/main/java/space/bxteam/divinemc/region/AbstractRegionFileFactory.java -@@ -0,0 +1,22 @@ -+package space.bxteam.divinemc.region; -+ -+import java.io.IOException; -+import java.nio.file.Path; -+import net.minecraft.world.level.chunk.storage.RegionFile; -+import net.minecraft.world.level.chunk.storage.RegionFileVersion; -+import net.minecraft.world.level.chunk.storage.RegionStorageInfo; -+ -+public class AbstractRegionFileFactory { -+ -+ public static AbstractRegionFile getAbstractRegionFile(RegionStorageInfo storageKey, Path directory, Path path, boolean dsync) throws IOException { -+ return getAbstractRegionFile(storageKey, directory, path, RegionFileVersion.getCompressionFormat(), dsync); -+ } -+ -+ public static AbstractRegionFile getAbstractRegionFile(RegionStorageInfo storageKey, Path path, Path directory, RegionFileVersion compressionFormat, boolean dsync) throws IOException { -+ if (path.toString().endsWith(".linear")) { -+ return new LinearRegionFile(path, storageKey.linearCompressionLevel()); -+ } else { -+ return new RegionFile(storageKey, path, directory, compressionFormat, dsync); -+ } -+ } -+} -diff --git a/src/main/java/space/bxteam/divinemc/region/LinearRegionFile.java b/src/main/java/space/bxteam/divinemc/region/LinearRegionFile.java -new file mode 100644 -index 0000000000000000000000000000000000000000..bcb69f307a63ae6aa2b1e8bdfbb1a499ad88d402 ---- /dev/null -+++ b/src/main/java/space/bxteam/divinemc/region/LinearRegionFile.java -@@ -0,0 +1,304 @@ -+package space.bxteam.divinemc.region; -+ -+import com.github.luben.zstd.ZstdInputStream; -+import com.github.luben.zstd.ZstdOutputStream; -+import com.mojang.logging.LogUtils; -+import java.io.BufferedOutputStream; -+import java.io.ByteArrayInputStream; -+import java.io.ByteArrayOutputStream; -+import java.io.DataInputStream; -+import java.io.DataOutputStream; -+import java.io.File; -+import java.io.FileInputStream; -+import java.io.FileOutputStream; -+import java.io.IOException; -+import java.io.InputStream; -+import java.nio.ByteBuffer; -+import java.nio.file.Files; -+import java.nio.file.Path; -+import java.nio.file.StandardCopyOption; -+import java.util.ArrayList; -+import java.util.Arrays; -+import java.util.List; -+import java.util.concurrent.atomic.AtomicBoolean; -+import javax.annotation.Nullable; -+import net.jpountz.lz4.LZ4Compressor; -+import net.jpountz.lz4.LZ4Factory; -+import net.jpountz.lz4.LZ4FastDecompressor; -+import net.minecraft.nbt.CompoundTag; -+import net.minecraft.world.level.ChunkPos; -+import org.slf4j.Logger; -+ -+public class LinearRegionFile implements AbstractRegionFile, AutoCloseable { -+ -+ private static final long SUPERBLOCK = -4323716122432332390L; -+ private static final byte VERSION = 2; -+ private static final int HEADER_SIZE = 32; -+ private static final int FOOTER_SIZE = 8; -+ private static final Logger LOGGER = LogUtils.getLogger(); -+ private static final List SUPPORTED_VERSIONS = Arrays.asList((byte) 1, (byte) 2); -+ private static final LinearRegionFileFlusher linearRegionFileFlusher = new LinearRegionFileFlusher(); -+ private final byte[][] buffer = new byte[1024][]; -+ private final int[] bufferUncompressedSize = new int[1024]; -+ private final int[] chunkTimestamps = new int[1024]; -+ private final LZ4Compressor compressor; -+ private final LZ4FastDecompressor decompressor; -+ private final int compressionLevel; -+ private final AtomicBoolean markedToSave = new AtomicBoolean(false); -+ public boolean closed = false; -+ public Path path; -+ -+ -+ public LinearRegionFile(Path file, int compression) throws IOException { -+ this.path = file; -+ this.compressionLevel = compression; -+ this.compressor = LZ4Factory.fastestInstance().fastCompressor(); -+ this.decompressor = LZ4Factory.fastestInstance().fastDecompressor(); -+ -+ File regionFile = new File(this.path.toString()); -+ -+ Arrays.fill(this.bufferUncompressedSize, 0); -+ -+ if (!regionFile.canRead()) return; -+ -+ try (FileInputStream fileStream = new FileInputStream(regionFile); -+ DataInputStream rawDataStream = new DataInputStream(fileStream)) { -+ -+ long superBlock = rawDataStream.readLong(); -+ if (superBlock != SUPERBLOCK) -+ throw new RuntimeException("Invalid superblock: " + superBlock + " in " + file); -+ -+ byte version = rawDataStream.readByte(); -+ if (!SUPPORTED_VERSIONS.contains(version)) -+ throw new RuntimeException("Invalid version: " + version + " in " + file); -+ -+ // Skip newestTimestamp (Long) + Compression level (Byte) + Chunk count (Short): Unused. -+ rawDataStream.skipBytes(11); -+ -+ int dataCount = rawDataStream.readInt(); -+ long fileLength = file.toFile().length(); -+ if (fileLength != HEADER_SIZE + dataCount + FOOTER_SIZE) -+ throw new IOException("Invalid file length: " + this.path + " " + fileLength + " " + (HEADER_SIZE + dataCount + FOOTER_SIZE)); -+ -+ rawDataStream.skipBytes(8); // Skip data hash (Long): Unused. -+ -+ byte[] rawCompressed = new byte[dataCount]; -+ rawDataStream.readFully(rawCompressed, 0, dataCount); -+ -+ superBlock = rawDataStream.readLong(); -+ if (superBlock != SUPERBLOCK) -+ throw new IOException("Footer superblock invalid " + this.path); -+ -+ try (DataInputStream dataStream = new DataInputStream(new ZstdInputStream(new ByteArrayInputStream(rawCompressed)))) { -+ -+ int[] starts = new int[1024]; -+ for (int i = 0; i < 1024; i++) { -+ starts[i] = dataStream.readInt(); -+ dataStream.skipBytes(4); // Skip timestamps (Int): Unused. -+ } -+ -+ for (int i = 0; i < 1024; i++) { -+ if (starts[i] > 0) { -+ int size = starts[i]; -+ byte[] b = new byte[size]; -+ dataStream.readFully(b, 0, size); -+ -+ int maxCompressedLength = this.compressor.maxCompressedLength(size); -+ byte[] compressed = new byte[maxCompressedLength]; -+ int compressedLength = this.compressor.compress(b, 0, size, compressed, 0, maxCompressedLength); -+ b = new byte[compressedLength]; -+ System.arraycopy(compressed, 0, b, 0, compressedLength); -+ -+ this.buffer[i] = b; -+ this.bufferUncompressedSize[i] = size; -+ } -+ } -+ } -+ } -+ } -+ -+ private static int getChunkIndex(int x, int z) { -+ return (x & 31) + ((z & 31) << 5); -+ } -+ -+ private static int getTimestamp() { -+ return (int) (System.currentTimeMillis() / 1000L); -+ } -+ -+ public Path getPath() { -+ return this.path; -+ } -+ -+ public void flush() throws IOException { -+ if (isMarkedToSave()) flushWrapper(); // sync -+ } -+ -+ private void markToSave() { -+ linearRegionFileFlusher.scheduleSave(this); -+ markedToSave.set(true); -+ } -+ -+ public boolean isMarkedToSave() { -+ return markedToSave.getAndSet(false); -+ } -+ -+ public void flushWrapper() { -+ try { -+ save(); -+ } catch (IOException e) { -+ LOGGER.error("Failed to flush region file {}", path.toAbsolutePath(), e); -+ } -+ } -+ -+ public boolean doesChunkExist(ChunkPos pos) { -+ return false; -+ } -+ -+ private synchronized void save() throws IOException { -+ long timestamp = getTimestamp(); -+ short chunkCount = 0; -+ -+ File tempFile = new File(path.toString() + ".tmp"); -+ -+ try (FileOutputStream fileStream = new FileOutputStream(tempFile); -+ ByteArrayOutputStream zstdByteArray = new ByteArrayOutputStream(); -+ ZstdOutputStream zstdStream = new ZstdOutputStream(zstdByteArray, this.compressionLevel); -+ DataOutputStream zstdDataStream = new DataOutputStream(zstdStream); -+ DataOutputStream dataStream = new DataOutputStream(fileStream)) { -+ -+ dataStream.writeLong(SUPERBLOCK); -+ dataStream.writeByte(VERSION); -+ dataStream.writeLong(timestamp); -+ dataStream.writeByte(this.compressionLevel); -+ -+ ArrayList byteBuffers = new ArrayList<>(); -+ for (int i = 0; i < 1024; i++) { -+ if (this.bufferUncompressedSize[i] != 0) { -+ chunkCount += 1; -+ byte[] content = new byte[bufferUncompressedSize[i]]; -+ this.decompressor.decompress(buffer[i], 0, content, 0, bufferUncompressedSize[i]); -+ -+ byteBuffers.add(content); -+ } else byteBuffers.add(null); -+ } -+ for (int i = 0; i < 1024; i++) { -+ zstdDataStream.writeInt(this.bufferUncompressedSize[i]); // Write uncompressed size -+ zstdDataStream.writeInt(this.chunkTimestamps[i]); // Write timestamp -+ } -+ for (int i = 0; i < 1024; i++) { -+ if (byteBuffers.get(i) != null) -+ zstdDataStream.write(byteBuffers.get(i), 0, byteBuffers.get(i).length); -+ } -+ zstdDataStream.close(); -+ -+ dataStream.writeShort(chunkCount); -+ -+ byte[] compressed = zstdByteArray.toByteArray(); -+ -+ dataStream.writeInt(compressed.length); -+ dataStream.writeLong(0); -+ -+ dataStream.write(compressed, 0, compressed.length); -+ dataStream.writeLong(SUPERBLOCK); -+ -+ dataStream.flush(); -+ fileStream.getFD().sync(); -+ fileStream.getChannel().force(true); // Ensure atomicity on Btrfs -+ } -+ Files.move(tempFile.toPath(), this.path, StandardCopyOption.REPLACE_EXISTING); -+ } -+ -+ public synchronized void write(ChunkPos pos, ByteBuffer buffer) { -+ try { -+ byte[] b = toByteArray(new ByteArrayInputStream(buffer.array())); -+ int uncompressedSize = b.length; -+ -+ int maxCompressedLength = this.compressor.maxCompressedLength(b.length); -+ byte[] compressed = new byte[maxCompressedLength]; -+ int compressedLength = this.compressor.compress(b, 0, b.length, compressed, 0, maxCompressedLength); -+ b = new byte[compressedLength]; -+ System.arraycopy(compressed, 0, b, 0, compressedLength); -+ -+ int index = getChunkIndex(pos.x, pos.z); -+ this.buffer[index] = b; -+ this.chunkTimestamps[index] = getTimestamp(); -+ this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] = uncompressedSize; -+ } catch (IOException e) { -+ LOGGER.error("Chunk write IOException {} {}", e, this.path); -+ } -+ markToSave(); -+ } -+ -+ public DataOutputStream getChunkDataOutputStream(ChunkPos pos) { -+ return new DataOutputStream(new BufferedOutputStream(new ChunkBuffer(pos))); -+ } -+ -+ private byte[] toByteArray(InputStream in) throws IOException { -+ ByteArrayOutputStream out = new ByteArrayOutputStream(); -+ byte[] tempBuffer = new byte[4096]; -+ -+ int length; -+ while ((length = in.read(tempBuffer)) >= 0) { -+ out.write(tempBuffer, 0, length); -+ } -+ -+ return out.toByteArray(); -+ } -+ -+ @Nullable -+ public synchronized DataInputStream getChunkDataInputStream(ChunkPos pos) { -+ if (this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] != 0) { -+ byte[] content = new byte[bufferUncompressedSize[getChunkIndex(pos.x, pos.z)]]; -+ this.decompressor.decompress(this.buffer[getChunkIndex(pos.x, pos.z)], 0, content, 0, bufferUncompressedSize[getChunkIndex(pos.x, pos.z)]); -+ return new DataInputStream(new ByteArrayInputStream(content)); -+ } -+ return null; -+ } -+ -+ public void clear(ChunkPos pos) { -+ int i = getChunkIndex(pos.x, pos.z); -+ this.buffer[i] = null; -+ this.bufferUncompressedSize[i] = 0; -+ this.chunkTimestamps[i] = getTimestamp(); -+ markToSave(); -+ } -+ -+ public boolean hasChunk(ChunkPos pos) { -+ return this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] > 0; -+ } -+ -+ public void close() throws IOException { -+ if (closed) return; -+ closed = true; -+ flush(); // sync -+ } -+ -+ public boolean recalculateHeader() { -+ return false; -+ } -+ -+ public void setOversized(int x, int z, boolean something) { -+ } -+ -+ public CompoundTag getOversizedData(int x, int z) throws IOException { -+ throw new IOException("getOversizedData is a stub " + this.path); -+ } -+ -+ public boolean isOversized(int x, int z) { -+ return false; -+ } -+ -+ private class ChunkBuffer extends ByteArrayOutputStream { -+ private final ChunkPos pos; -+ -+ public ChunkBuffer(ChunkPos chunkcoordintpair) { -+ super(); -+ this.pos = chunkcoordintpair; -+ } -+ -+ public void close() { -+ ByteBuffer bytebuffer = ByteBuffer.wrap(this.buf, 0, this.count); -+ LinearRegionFile.this.write(this.pos, bytebuffer); -+ } -+ } -+} -diff --git a/src/main/java/space/bxteam/divinemc/region/LinearRegionFileFlusher.java b/src/main/java/space/bxteam/divinemc/region/LinearRegionFileFlusher.java -new file mode 100644 -index 0000000000000000000000000000000000000000..731dd3516a597e9cb97596bb7647db6c94363a9a ---- /dev/null -+++ b/src/main/java/space/bxteam/divinemc/region/LinearRegionFileFlusher.java -@@ -0,0 +1,50 @@ -+package space.bxteam.divinemc.region; -+ -+import com.google.common.util.concurrent.ThreadFactoryBuilder; -+import java.util.Queue; -+import java.util.concurrent.ExecutorService; -+import java.util.concurrent.Executors; -+import java.util.concurrent.LinkedBlockingQueue; -+import java.util.concurrent.ScheduledExecutorService; -+import java.util.concurrent.TimeUnit; -+import org.bukkit.Bukkit; -+import space.bxteam.divinemc.configuration.DivineConfig; -+ -+public class LinearRegionFileFlusher { -+ -+ private final Queue savingQueue = new LinkedBlockingQueue<>(); -+ private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor( -+ new ThreadFactoryBuilder() -+ .setNameFormat("linear-flush-scheduler") -+ .build() -+ ); -+ private final ExecutorService executor = Executors.newFixedThreadPool( -+ DivineConfig.linearFlushThreads, -+ new ThreadFactoryBuilder() -+ .setNameFormat("linear-flusher-%d") -+ .build() -+ ); -+ -+ public LinearRegionFileFlusher() { -+ Bukkit.getLogger().info("Using " + DivineConfig.linearFlushThreads + " threads for linear region flushing."); -+ scheduler.scheduleAtFixedRate(this::pollAndFlush, 0L, DivineConfig.linearFlushFrequency, TimeUnit.SECONDS); -+ } -+ -+ public void scheduleSave(LinearRegionFile regionFile) { -+ if (savingQueue.contains(regionFile)) return; -+ savingQueue.add(regionFile); -+ } -+ -+ private void pollAndFlush() { -+ while (!savingQueue.isEmpty()) { -+ LinearRegionFile regionFile = savingQueue.poll(); -+ if (!regionFile.closed && regionFile.isMarkedToSave()) -+ executor.execute(regionFile::flushWrapper); -+ } -+ } -+ -+ public void shutdown() { -+ executor.shutdown(); -+ scheduler.shutdown(); -+ } -+} -diff --git a/src/main/java/space/bxteam/divinemc/region/RegionFileFormat.java b/src/main/java/space/bxteam/divinemc/region/RegionFileFormat.java -new file mode 100644 -index 0000000000000000000000000000000000000000..a799c9b2bb262b86be3872ba2a920ca3e8cb9d02 ---- /dev/null -+++ b/src/main/java/space/bxteam/divinemc/region/RegionFileFormat.java -@@ -0,0 +1,16 @@ -+package space.bxteam.divinemc.region; -+ -+public enum RegionFileFormat { -+ ANVIL, -+ LINEAR, -+ INVALID; -+ -+ public static RegionFileFormat fromString(String format) { -+ for (RegionFileFormat rff : values()) { -+ if (rff.name().equalsIgnoreCase(format)) { -+ return rff; -+ } -+ } -+ return RegionFileFormat.INVALID; -+ } -+} diff --git a/divinemc-server/build.gradle.kts.patch b/divinemc-server/build.gradle.kts.patch index f332c3a..a1103cb 100644 --- a/divinemc-server/build.gradle.kts.patch +++ b/divinemc-server/build.gradle.kts.patch @@ -57,6 +57,16 @@ implementation("ca.spottedleaf:concurrentutil:0.0.3") implementation("org.jline:jline-terminal-ffm:3.27.1") // use ffm on java 22+ implementation("org.jline:jline-terminal-jni:3.27.1") // fall back to jni on java 21 +@@ -176,6 +_,9 @@ + implementation("org.mozilla:rhino-engine:1.7.14") // Purpur + implementation("dev.omega24:upnp4j:1.0") // Purpur + ++ implementation("com.github.luben:zstd-jni:1.5.6-9") // DivineMC ++ implementation("org.lz4:lz4-java:1.8.0") // DivineMC ++ + runtimeOnly("org.apache.maven:maven-resolver-provider:3.9.6") + runtimeOnly("org.apache.maven.resolver:maven-resolver-connector-basic:1.9.18") + runtimeOnly("org.apache.maven.resolver:maven-resolver-transport-http:1.9.18") @@ -203,26 +_,35 @@ implementation("me.lucko:spark-paper:1.10.119-SNAPSHOT") } diff --git a/divinemc-server/minecraft-patches/features/0013-Implement-Linear-region-format.patch b/divinemc-server/minecraft-patches/features/0013-Implement-Linear-region-format.patch new file mode 100644 index 0000000..bea8f07 --- /dev/null +++ b/divinemc-server/minecraft-patches/features/0013-Implement-Linear-region-format.patch @@ -0,0 +1,362 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: NONPLAYT <76615486+NONPLAYT@users.noreply.github.com> +Date: Sat, 18 Jan 2025 01:00:23 +0300 +Subject: [PATCH] Implement Linear region format + + +diff --git a/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java b/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java +index a814512fcfb85312474ae2c2c21443843bf57831..a92f4bce3b599b25db821bc921fbac3f437cb4a1 100644 +--- a/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java ++++ b/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java +@@ -8,9 +8,9 @@ public interface ChunkSystemRegionFileStorage { + + public boolean moonrise$doesRegionFileNotExistNoIO(final int chunkX, final int chunkZ); + +- public RegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ); ++ public space.bxteam.divinemc.region.AbstractRegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ); // DivineMC - Linear Region format + +- public RegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException; ++ public space.bxteam.divinemc.region.AbstractRegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException; // DivineMC - Linear Region format + + public MoonriseRegionFileIO.RegionDataController.WriteData moonrise$startWrite( + final int chunkX, final int chunkZ, final CompoundTag compound +diff --git a/ca/spottedleaf/moonrise/patches/chunk_system/io/MoonriseRegionFileIO.java b/ca/spottedleaf/moonrise/patches/chunk_system/io/MoonriseRegionFileIO.java +index 1acea58838f057ab87efd103cbecb6f5aeaef393..24e1b9dbee078341bd57794e06f00bd9ebb874d3 100644 +--- a/ca/spottedleaf/moonrise/patches/chunk_system/io/MoonriseRegionFileIO.java ++++ b/ca/spottedleaf/moonrise/patches/chunk_system/io/MoonriseRegionFileIO.java +@@ -1462,7 +1462,7 @@ public final class MoonriseRegionFileIO { + + public static interface IORunnable { + +- public void run(final RegionFile regionFile) throws IOException; ++ public void run(final space.bxteam.divinemc.region.AbstractRegionFile regionFile) throws IOException; // DivineMC - linear region + + } + } +diff --git a/ca/spottedleaf/moonrise/patches/chunk_system/storage/ChunkSystemChunkBuffer.java b/ca/spottedleaf/moonrise/patches/chunk_system/storage/ChunkSystemChunkBuffer.java +index 51c126735ace8fdde89ad97b5cab62f244212db0..44e3329e2429da20cdc94b7779836b9baf9be087 100644 +--- a/ca/spottedleaf/moonrise/patches/chunk_system/storage/ChunkSystemChunkBuffer.java ++++ b/ca/spottedleaf/moonrise/patches/chunk_system/storage/ChunkSystemChunkBuffer.java +@@ -8,5 +8,5 @@ public interface ChunkSystemChunkBuffer { + + public void moonrise$setWriteOnClose(final boolean value); + +- public void moonrise$write(final RegionFile regionFile) throws IOException; ++ public void moonrise$write(final space.bxteam.divinemc.region.AbstractRegionFile regionFile) throws IOException; // DivineMC - linear region + } +diff --git a/net/minecraft/server/MinecraftServer.java b/net/minecraft/server/MinecraftServer.java +index d04c06fafd133f773f311e7c2708fa8b049da67c..56895a283cc1183bcc15051236085aabf7836f2e 100644 +--- a/net/minecraft/server/MinecraftServer.java ++++ b/net/minecraft/server/MinecraftServer.java +@@ -950,10 +950,10 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop> progressMap = Reference2FloatMaps.synchronize(new Reference2FloatOpenHashMap<>()); + volatile Component status = Component.translatable("optimizeWorld.stage.counting"); +- static final Pattern REGEX = Pattern.compile("^r\\.(-?[0-9]+)\\.(-?[0-9]+)\\.mca$"); ++ static final Pattern REGEX = Pattern.compile("^r\\.(-?[0-9]+)\\.(-?[0-9]+)\\.(linear | mca)$"); // DivineMC - linear region + final DimensionDataStorage overworldDataStorage; + + public WorldUpgrader( +@@ -261,7 +261,11 @@ public class WorldUpgrader implements AutoCloseable { + } + + private static List getAllChunkPositions(RegionStorageInfo regionStorageInfo, Path path) { +- File[] files = path.toFile().listFiles((directory, filename) -> filename.endsWith(".mca")); ++ // DivineMC start - linear region ++ File[] files = path.toFile().listFiles((directory, filename) -> { ++ return filename.endsWith(".linear") || filename.endsWith(".mca"); ++ }); ++ // DivineMC end - linear region + if (files == null) { + return List.of(); + } else { +@@ -274,7 +278,8 @@ public class WorldUpgrader implements AutoCloseable { + int i1 = Integer.parseInt(matcher.group(2)) << 5; + List list1 = Lists.newArrayList(); + +- try (RegionFile regionFile = new RegionFile(regionStorageInfo, file.toPath(), path, true)) { ++ try { ++ space.bxteam.divinemc.region.AbstractRegionFile regionFile = space.bxteam.divinemc.region.AbstractRegionFileFactory.getAbstractRegionFile(regionStorageInfo, file.toPath(), path, true); // DivineMC - linear region format + for (int i2 = 0; i2 < 32; i2++) { + for (int i3 = 0; i3 < 32; i3++) { + ChunkPos chunkPos = new ChunkPos(i2 + i, i3 + i1); +@@ -322,7 +327,7 @@ public class WorldUpgrader implements AutoCloseable { + + protected abstract boolean tryProcessOnePosition(T chunkStorage, ChunkPos chunkPos, ResourceKey dimension); + +- private void onFileFinished(RegionFile regionFile) { ++ private void onFileFinished(space.bxteam.divinemc.region.AbstractRegionFile regionFile) { // DivineMC - linear region + if (WorldUpgrader.this.recreateRegionFiles) { + if (this.previousWriteFuture != null) { + this.previousWriteFuture.join(); +@@ -424,7 +429,7 @@ public class WorldUpgrader implements AutoCloseable { + } + } + +- record FileToUpgrade(RegionFile file, List chunksToUpgrade) { ++ record FileToUpgrade(space.bxteam.divinemc.region.AbstractRegionFile file, List chunksToUpgrade) { // DivineMC - linear region + } + + class PoiUpgrader extends WorldUpgrader.SimpleRegionStorageUpgrader { +diff --git a/net/minecraft/world/level/chunk/storage/RegionFile.java b/net/minecraft/world/level/chunk/storage/RegionFile.java +index c72494e757a9dc50e053dbc873f7b30e83d5cb8c..1490df47f0d133f14c4cb10c901ac80b0be429a1 100644 +--- a/net/minecraft/world/level/chunk/storage/RegionFile.java ++++ b/net/minecraft/world/level/chunk/storage/RegionFile.java +@@ -22,7 +22,7 @@ import net.minecraft.util.profiling.jfr.JvmProfiler; + import net.minecraft.world.level.ChunkPos; + import org.slf4j.Logger; + +-public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patches.chunk_system.storage.ChunkSystemRegionFile { // Paper - rewrite chunk system ++public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patches.chunk_system.storage.ChunkSystemRegionFile, space.bxteam.divinemc.region.AbstractRegionFile { // Paper - rewrite chunk system // DivineMC - Linear region file + private static final Logger LOGGER = LogUtils.getLogger(); + public static final int MAX_CHUNK_SIZE = 500 * 1024 * 1024; // Paper - don't write garbage data to disk if writing serialization fails + private static final int SECTOR_BYTES = 4096; +@@ -904,7 +904,7 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche + } + + @Override +- public final void moonrise$write(final RegionFile regionFile) throws IOException { ++ public final void moonrise$write(final space.bxteam.divinemc.region.AbstractRegionFile regionFile) throws IOException { // DivineMC - linear region + regionFile.write(this.pos, ByteBuffer.wrap(this.buf, 0, this.count)); + } + // Paper end - rewrite chunk system +diff --git a/net/minecraft/world/level/chunk/storage/RegionFileStorage.java b/net/minecraft/world/level/chunk/storage/RegionFileStorage.java +index 6ebd1300c2561116b83cb2472ac7939ead36d576..4b8062c3335a73f6f8ef5ae0f9f0f2a527f959de 100644 +--- a/net/minecraft/world/level/chunk/storage/RegionFileStorage.java ++++ b/net/minecraft/world/level/chunk/storage/RegionFileStorage.java +@@ -18,7 +18,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + private static final org.slf4j.Logger LOGGER = com.mojang.logging.LogUtils.getLogger(); // Paper + public static final String ANVIL_EXTENSION = ".mca"; + private static final int MAX_CACHE_SIZE = 256; +- public final Long2ObjectLinkedOpenHashMap regionCache = new Long2ObjectLinkedOpenHashMap<>(); ++ public final Long2ObjectLinkedOpenHashMap regionCache = new Long2ObjectLinkedOpenHashMap(); // DivineMC - linear region + private final RegionStorageInfo info; + private final Path folder; + private final boolean sync; +@@ -33,7 +33,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + @Nullable + public static ChunkPos getRegionFileCoordinates(Path file) { + String fileName = file.getFileName().toString(); +- if (!fileName.startsWith("r.") || !fileName.endsWith(".mca")) { ++ if (!fileName.startsWith("r.") || !fileName.endsWith(".mca") || !fileName.endsWith(".linear")) { + return null; + } + +@@ -57,9 +57,13 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + private static final int REGION_SHIFT = 5; + private static final int MAX_NON_EXISTING_CACHE = 1024 * 4; + private final it.unimi.dsi.fastutil.longs.LongLinkedOpenHashSet nonExistingRegionFiles = new it.unimi.dsi.fastutil.longs.LongLinkedOpenHashSet(); +- private static String getRegionFileName(final int chunkX, final int chunkZ) { ++ // DivineMC start - linear region format ++ private static String getRegionFileName(final RegionStorageInfo info, final int chunkX, final int chunkZ) { ++ if (info.regionFormat().equals(space.bxteam.divinemc.region.RegionFileFormat.LINEAR)) ++ return "r." + (chunkX >> REGION_SHIFT) + "." + (chunkZ >> REGION_SHIFT) + ".linear"; + return "r." + (chunkX >> REGION_SHIFT) + "." + (chunkZ >> REGION_SHIFT) + ".mca"; + } ++ // DivineMC end - linear region format + + private boolean doesRegionFilePossiblyExist(final long position) { + synchronized (this.nonExistingRegionFiles) { +@@ -93,15 +97,15 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + } + + @Override +- public synchronized final RegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ) { ++ public synchronized final space.bxteam.divinemc.region.AbstractRegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ) { // DivineMC - linear region + return this.regionCache.getAndMoveToFirst(ChunkPos.asLong(chunkX >> REGION_SHIFT, chunkZ >> REGION_SHIFT)); + } + + @Override +- public synchronized final RegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException { ++ public synchronized final space.bxteam.divinemc.region.AbstractRegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException { // DivineMC - linear region + final long key = ChunkPos.asLong(chunkX >> REGION_SHIFT, chunkZ >> REGION_SHIFT); + +- RegionFile ret = this.regionCache.getAndMoveToFirst(key); ++ space.bxteam.divinemc.region.AbstractRegionFile ret = this.regionCache.getAndMoveToFirst(key); // DivineMC - linear region + if (ret != null) { + return ret; + } +@@ -114,7 +118,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + this.regionCache.removeLast().close(); + } + +- final Path regionPath = this.folder.resolve(getRegionFileName(chunkX, chunkZ)); ++ final Path regionPath = this.folder.resolve(getRegionFileName(this.info, chunkX, chunkZ)); // DivineMC - linear region + + if (!java.nio.file.Files.exists(regionPath)) { + this.markNonExisting(key); +@@ -125,7 +129,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + + FileUtil.createDirectoriesSafe(this.folder); + +- ret = new RegionFile(this.info, regionPath, this.folder, this.sync); ++ ret = space.bxteam.divinemc.region.AbstractRegionFileFactory.getAbstractRegionFile(this.info, regionPath, this.folder, this.sync); // DivineMC - linear region + + this.regionCache.putAndMoveToFirst(key, ret); + +@@ -144,7 +148,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + } + + final ChunkPos pos = new ChunkPos(chunkX, chunkZ); +- final RegionFile regionFile = this.getRegionFile(pos); ++ final space.bxteam.divinemc.region.AbstractRegionFile regionFile = this.getRegionFile(pos); // DivineMC - linear region + + // note: not required to keep regionfile loaded after this call, as the write param takes a regionfile as input + // (and, the regionfile parameter is unused for writing until the write call) +@@ -178,7 +182,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + ) throws IOException { + final ChunkPos pos = new ChunkPos(chunkX, chunkZ); + if (writeData.result() == ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO.RegionDataController.WriteData.WriteResult.DELETE) { +- final RegionFile regionFile = this.moonrise$getRegionFileIfExists(chunkX, chunkZ); ++ final space.bxteam.divinemc.region.AbstractRegionFile regionFile = this.moonrise$getRegionFileIfExists(chunkX, chunkZ); // DivineMC - linear region + if (regionFile != null) { + regionFile.clear(pos); + } // else: didn't exist +@@ -193,7 +197,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + public final ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO.RegionDataController.ReadData moonrise$readData( + final int chunkX, final int chunkZ + ) throws IOException { +- final RegionFile regionFile = this.moonrise$getRegionFileIfExists(chunkX, chunkZ); ++ final space.bxteam.divinemc.region.AbstractRegionFile regionFile = this.moonrise$getRegionFileIfExists(chunkX, chunkZ); // DivineMC - linear region + + final DataInputStream input = regionFile == null ? null : regionFile.getChunkDataInputStream(new ChunkPos(chunkX, chunkZ)); + +@@ -237,7 +241,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + } + // Paper end - rewrite chunk system + // Paper start - rewrite chunk system +- public RegionFile getRegionFile(ChunkPos chunkcoordintpair) throws IOException { ++ public space.bxteam.divinemc.region.AbstractRegionFile getRegionFile(ChunkPos chunkcoordintpair) throws IOException { // DivineMC - linear region + return this.getRegionFile(chunkcoordintpair, false); + } + // Paper end - rewrite chunk system +@@ -249,7 +253,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + this.isChunkData = isChunkDataFolder(this.folder); // Paper - recalculate region file headers + } + +- @org.jetbrains.annotations.Contract("_, false -> !null") @Nullable private RegionFile getRegionFile(ChunkPos chunkPos, boolean existingOnly) throws IOException { // CraftBukkit ++ @org.jetbrains.annotations.Contract("_, false -> !null") @Nullable private space.bxteam.divinemc.region.AbstractRegionFile getRegionFile(ChunkPos chunkPos, boolean existingOnly) throws IOException { // CraftBukkit // DivineMC - linear region + // Paper start - rewrite chunk system + if (existingOnly) { + return this.moonrise$getRegionFileIfExists(chunkPos.x, chunkPos.z); +@@ -257,7 +261,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + synchronized (this) { + final long key = ChunkPos.asLong(chunkPos.x >> REGION_SHIFT, chunkPos.z >> REGION_SHIFT); + +- RegionFile ret = this.regionCache.getAndMoveToFirst(key); ++ space.bxteam.divinemc.region.AbstractRegionFile ret = this.regionCache.getAndMoveToFirst(key); // DivineMC - linear region + if (ret != null) { + return ret; + } +@@ -266,13 +270,13 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + this.regionCache.removeLast().close(); + } + +- final Path regionPath = this.folder.resolve(getRegionFileName(chunkPos.x, chunkPos.z)); ++ final Path regionPath = this.folder.resolve(getRegionFileName(this.info, chunkPos.x, chunkPos.z)); // DivineMC - linear region + + this.createRegionFile(key); + + FileUtil.createDirectoriesSafe(this.folder); + +- ret = new RegionFile(this.info, regionPath, this.folder, this.sync); ++ ret = space.bxteam.divinemc.region.AbstractRegionFileFactory.getAbstractRegionFile(this.info, regionPath, this.folder, this.sync); // DivineMC - linear region + + this.regionCache.putAndMoveToFirst(key, ret); + +@@ -286,7 +290,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + org.apache.logging.log4j.LogManager.getLogger().fatal(msg + " (" + file.toString().replaceAll(".+[\\\\/]", "") + " - " + x + "," + z + ") Go clean it up to remove this message. /minecraft:tp " + (x<<4)+" 128 "+(z<<4) + " - DO NOT REPORT THIS TO DIVINEMC - You may ask for help on Discord, but do not file an issue. These error messages can not be removed."); // DivineMC - Rebrand + } + +- private static CompoundTag readOversizedChunk(RegionFile regionfile, ChunkPos chunkCoordinate) throws IOException { ++ private static CompoundTag readOversizedChunk(space.bxteam.divinemc.region.AbstractRegionFile regionfile, ChunkPos chunkCoordinate) throws IOException { // DivineMC - linear region + synchronized (regionfile) { + try (DataInputStream datainputstream = regionfile.getChunkDataInputStream(chunkCoordinate)) { + CompoundTag oversizedData = regionfile.getOversizedData(chunkCoordinate.x, chunkCoordinate.z); +@@ -321,7 +325,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + @Nullable + public CompoundTag read(ChunkPos chunkPos) throws IOException { + // CraftBukkit start - SPIGOT-5680: There's no good reason to preemptively create files on read, save that for writing +- RegionFile regionFile = this.getRegionFile(chunkPos, true); ++ space.bxteam.divinemc.region.AbstractRegionFile regionFile = this.getRegionFile(chunkPos, true); // DivineMC - linear region + if (regionFile == null) { + return null; + } +@@ -360,7 +364,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + + public void scanChunk(ChunkPos chunkPos, StreamTagVisitor visitor) throws IOException { + // CraftBukkit start - SPIGOT-5680: There's no good reason to preemptively create files on read, save that for writing +- RegionFile regionFile = this.getRegionFile(chunkPos, true); ++ space.bxteam.divinemc.region.AbstractRegionFile regionFile = this.getRegionFile(chunkPos, true); // DivineMC - linear region + if (regionFile == null) { + return; + } +@@ -374,7 +378,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + } + + public void write(ChunkPos chunkPos, @Nullable CompoundTag chunkData) throws IOException { // Paper - rewrite chunk system - public +- RegionFile regionFile = this.getRegionFile(chunkPos, chunkData == null); // CraftBukkit // Paper - rewrite chunk system ++ space.bxteam.divinemc.region.AbstractRegionFile regionFile = this.getRegionFile(chunkPos, chunkData == null); // CraftBukkit // Paper - rewrite chunk system // DivineMC - linear region + // Paper start - rewrite chunk system + if (regionFile == null) { + // if the RegionFile doesn't exist, no point in deleting from it +@@ -404,7 +408,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + // Paper start - rewrite chunk system + synchronized (this) { + final ExceptionCollector exceptionCollector = new ExceptionCollector<>(); +- for (final RegionFile regionFile : this.regionCache.values()) { ++ for (final space.bxteam.divinemc.region.AbstractRegionFile regionFile : this.regionCache.values()) { // DivineMC - linear region + try { + regionFile.close(); + } catch (final IOException ex) { +@@ -420,7 +424,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + // Paper start - rewrite chunk system + synchronized (this) { + final ExceptionCollector exceptionCollector = new ExceptionCollector<>(); +- for (final RegionFile regionFile : this.regionCache.values()) { ++ for (final space.bxteam.divinemc.region.AbstractRegionFile regionFile : this.regionCache.values()) { // DivineMC - linear region + try { + regionFile.flush(); + } catch (final IOException ex) { +diff --git a/net/minecraft/world/level/chunk/storage/RegionStorageInfo.java b/net/minecraft/world/level/chunk/storage/RegionStorageInfo.java +index 6111631c6673948b266286894603cc5e30451b02..123a1069346711237ad040657602fd2e11e16db0 100644 +--- a/net/minecraft/world/level/chunk/storage/RegionStorageInfo.java ++++ b/net/minecraft/world/level/chunk/storage/RegionStorageInfo.java +@@ -7,4 +7,20 @@ public record RegionStorageInfo(String level, ResourceKey dimension, Stri + public RegionStorageInfo withTypeSuffix(String suffix) { + return new RegionStorageInfo(this.level, this.dimension, this.type + suffix); + } ++ ++ // DivineMC start - Linear Region format ++ public space.bxteam.divinemc.region.RegionFileFormat regionFormat() { ++ return ((org.bukkit.craftbukkit.CraftWorld) org.bukkit.Bukkit.getWorld(level)) ++ .getHandle() ++ .divinemcConfig ++ .regionFormat; ++ } ++ ++ public int linearCompressionLevel() { ++ return ((org.bukkit.craftbukkit.CraftWorld) org.bukkit.Bukkit.getWorld(level)) ++ .getHandle() ++ .divinemcConfig ++ .linearCompressionLevel; ++ } ++ // DivineMC end - Linear Region format + } diff --git a/divinemc-server/src/main/java/space/bxteam/divinemc/configuration/DivineConfig.java b/divinemc-server/src/main/java/space/bxteam/divinemc/configuration/DivineConfig.java index a6b1b11..5ffd581 100644 --- a/divinemc-server/src/main/java/space/bxteam/divinemc/configuration/DivineConfig.java +++ b/divinemc-server/src/main/java/space/bxteam/divinemc/configuration/DivineConfig.java @@ -163,6 +163,18 @@ public class DivineConfig { enableSecureSeed = getBoolean("settings.misc.enable-secure-seed", enableSecureSeed); } + public static int linearFlushFrequency = 5; + public static int linearFlushThreads = 1; + private static void linearSettings() { + linearFlushFrequency = getInt("settings.region-format.linear.flush-frequency", linearFlushFrequency); + linearFlushThreads = getInt("settings.region-format.linear.flush-max-threads", linearFlushThreads); + + if (linearFlushThreads < 0) + linearFlushThreads = Math.max(Runtime.getRuntime().availableProcessors() + linearFlushThreads, 1); + else + linearFlushThreads = Math.max(linearFlushThreads, 1); + } + public static boolean asyncPathfinding = true; public static int asyncPathfindingMaxThreads = 0; public static int asyncPathfindingKeepalive = 60; diff --git a/divinemc-server/src/main/java/space/bxteam/divinemc/configuration/DivineWorldConfig.java b/divinemc-server/src/main/java/space/bxteam/divinemc/configuration/DivineWorldConfig.java index 8f0b2ce..3fa0bf2 100644 --- a/divinemc-server/src/main/java/space/bxteam/divinemc/configuration/DivineWorldConfig.java +++ b/divinemc-server/src/main/java/space/bxteam/divinemc/configuration/DivineWorldConfig.java @@ -3,10 +3,12 @@ package space.bxteam.divinemc.configuration; import org.apache.commons.lang.BooleanUtils; import org.bukkit.World; import org.bukkit.configuration.ConfigurationSection; +import space.bxteam.divinemc.region.RegionFileFormat; import java.util.List; import java.util.Map; import java.util.function.Predicate; +import java.util.logging.Level; import static space.bxteam.divinemc.configuration.DivineConfig.log; @@ -91,4 +93,22 @@ public class DivineWorldConfig { private void suppressErrorsFromDirtyAttributes() { suppressErrorsFromDirtyAttributes = getBoolean("suppress-errors-from-dirty-attributes", suppressErrorsFromDirtyAttributes); } + + public RegionFileFormat regionFormat = RegionFileFormat.ANVIL; + public int linearCompressionLevel = 1; + private void regionFormatSettings() { + regionFormat = RegionFileFormat.fromString(getString("region-format.format", regionFormat.name())); + if (regionFormat.equals(RegionFileFormat.INVALID)) { + log(Level.SEVERE, "Unknown region format in linear.yml: " + regionFormat); + log(Level.SEVERE, "Falling back to ANVIL region file format."); + regionFormat = RegionFileFormat.ANVIL; + } + + linearCompressionLevel = getInt("region-format.linear.compression-level", linearCompressionLevel); + if (linearCompressionLevel > 23 || linearCompressionLevel < 1) { + log(Level.SEVERE, "Linear region compression level should be between 1 and 22 in linear.yml: " + linearCompressionLevel); + log(Level.SEVERE, "Falling back to compression level 1."); + linearCompressionLevel = 1; + } + } } diff --git a/divinemc-server/src/main/java/space/bxteam/divinemc/region/AbstractRegionFile.java b/divinemc-server/src/main/java/space/bxteam/divinemc/region/AbstractRegionFile.java new file mode 100644 index 0000000..573fc90 --- /dev/null +++ b/divinemc-server/src/main/java/space/bxteam/divinemc/region/AbstractRegionFile.java @@ -0,0 +1,27 @@ +package space.bxteam.divinemc.region; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import net.minecraft.nbt.CompoundTag; +import net.minecraft.world.level.ChunkPos; + +public interface AbstractRegionFile { + Path getPath(); + void flush() throws IOException; + void clear(ChunkPos pos) throws IOException; + void close() throws IOException; + void setOversized(int x, int z, boolean b) throws IOException; + void write(ChunkPos pos, ByteBuffer buffer) throws IOException; + + boolean hasChunk(ChunkPos pos); + boolean doesChunkExist(ChunkPos pos) throws Exception; + boolean isOversized(int x, int z); + boolean recalculateHeader() throws IOException; + + DataOutputStream getChunkDataOutputStream(ChunkPos pos) throws IOException; + DataInputStream getChunkDataInputStream(ChunkPos pos) throws IOException; + CompoundTag getOversizedData(int x, int z) throws IOException; +} diff --git a/divinemc-server/src/main/java/space/bxteam/divinemc/region/AbstractRegionFileFactory.java b/divinemc-server/src/main/java/space/bxteam/divinemc/region/AbstractRegionFileFactory.java new file mode 100644 index 0000000..d9c2d34 --- /dev/null +++ b/divinemc-server/src/main/java/space/bxteam/divinemc/region/AbstractRegionFileFactory.java @@ -0,0 +1,25 @@ +package space.bxteam.divinemc.region; + +import java.io.IOException; +import java.nio.file.Path; +import net.minecraft.world.level.chunk.storage.RegionFile; +import net.minecraft.world.level.chunk.storage.RegionFileVersion; +import net.minecraft.world.level.chunk.storage.RegionStorageInfo; +import org.jetbrains.annotations.Contract; +import org.jetbrains.annotations.NotNull; + +public class AbstractRegionFileFactory { + @Contract("_, _, _, _ -> new") + public static @NotNull AbstractRegionFile getAbstractRegionFile(RegionStorageInfo storageKey, Path directory, Path path, boolean dsync) throws IOException { + return getAbstractRegionFile(storageKey, directory, path, RegionFileVersion.getCompressionFormat(), dsync); + } + + @Contract("_, _, _, _, _ -> new") + public static @NotNull AbstractRegionFile getAbstractRegionFile(RegionStorageInfo storageKey, Path path, Path directory, RegionFileVersion compressionFormat, boolean dsync) throws IOException { + if (path.toString().endsWith(".linear")) { + return new LinearRegionFile(path, storageKey.linearCompressionLevel()); + } else { + return new RegionFile(storageKey, path, directory, compressionFormat, dsync); + } + } +} diff --git a/divinemc-server/src/main/java/space/bxteam/divinemc/region/LinearRegionFile.java b/divinemc-server/src/main/java/space/bxteam/divinemc/region/LinearRegionFile.java new file mode 100644 index 0000000..bf11a41 --- /dev/null +++ b/divinemc-server/src/main/java/space/bxteam/divinemc/region/LinearRegionFile.java @@ -0,0 +1,298 @@ +package space.bxteam.divinemc.region; + +import com.github.luben.zstd.ZstdInputStream; +import com.github.luben.zstd.ZstdOutputStream; +import com.mojang.logging.LogUtils; +import java.io.BufferedOutputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; + +import net.jpountz.lz4.LZ4Compressor; +import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4FastDecompressor; +import net.minecraft.nbt.CompoundTag; +import net.minecraft.world.level.ChunkPos; +import org.slf4j.Logger; +import space.bxteam.divinemc.configuration.DivineConfig; + +public class LinearRegionFile implements AbstractRegionFile, AutoCloseable { + private static final long SUPERBLOCK = -4323716122432332390L; + private static final byte VERSION = 2; + private static final int HEADER_SIZE = 32; + private static final int FOOTER_SIZE = 8; + private static final Logger LOGGER = LogUtils.getLogger(); + private static final List SUPPORTED_VERSIONS = Arrays.asList((byte) 1, (byte) 2); + private final byte[][] buffer = new byte[1024][]; + private final int[] bufferUncompressedSize = new int[1024]; + private final int[] chunkTimestamps = new int[1024]; + private final LZ4Compressor compressor; + private final LZ4FastDecompressor decompressor; + private final int compressionLevel; + public boolean closed = false; + public Path path; + private volatile long lastFlushed = System.nanoTime(); + + public LinearRegionFile(Path file, int compression) throws IOException { + this.path = file; + this.compressionLevel = compression; + this.compressor = LZ4Factory.fastestInstance().fastCompressor(); + this.decompressor = LZ4Factory.fastestInstance().fastDecompressor(); + + File regionFile = new File(this.path.toString()); + + Arrays.fill(this.bufferUncompressedSize, 0); + + if (!regionFile.canRead()) return; + + try (FileInputStream fileStream = new FileInputStream(regionFile); + DataInputStream rawDataStream = new DataInputStream(fileStream)) { + + long superBlock = rawDataStream.readLong(); + if (superBlock != SUPERBLOCK) + throw new RuntimeException("Invalid superblock: " + superBlock + " in " + file); + + byte version = rawDataStream.readByte(); + if (!SUPPORTED_VERSIONS.contains(version)) + throw new RuntimeException("Invalid version: " + version + " in " + file); + + // Skip newestTimestamp (Long) + Compression level (Byte) + Chunk count (Short): Unused. + rawDataStream.skipBytes(11); + + int dataCount = rawDataStream.readInt(); + long fileLength = file.toFile().length(); + if (fileLength != HEADER_SIZE + dataCount + FOOTER_SIZE) + throw new IOException("Invalid file length: " + this.path + " " + fileLength + " " + (HEADER_SIZE + dataCount + FOOTER_SIZE)); + + rawDataStream.skipBytes(8); // Skip data hash (Long): Unused. + + byte[] rawCompressed = new byte[dataCount]; + rawDataStream.readFully(rawCompressed, 0, dataCount); + + superBlock = rawDataStream.readLong(); + if (superBlock != SUPERBLOCK) + throw new IOException("Footer superblock invalid " + this.path); + + try (DataInputStream dataStream = new DataInputStream(new ZstdInputStream(new ByteArrayInputStream(rawCompressed)))) { + + int[] starts = new int[1024]; + for (int i = 0; i < 1024; i++) { + starts[i] = dataStream.readInt(); + dataStream.skipBytes(4); // Skip timestamps (Int): Unused. + } + + for (int i = 0; i < 1024; i++) { + if (starts[i] > 0) { + int size = starts[i]; + byte[] b = new byte[size]; + dataStream.readFully(b, 0, size); + + int maxCompressedLength = this.compressor.maxCompressedLength(size); + byte[] compressed = new byte[maxCompressedLength]; + int compressedLength = this.compressor.compress(b, 0, size, compressed, 0, maxCompressedLength); + b = new byte[compressedLength]; + System.arraycopy(compressed, 0, b, 0, compressedLength); + + this.buffer[i] = b; + this.bufferUncompressedSize[i] = size; + } + } + } + } + } + + private static int getChunkIndex(int x, int z) { + return (x & 31) + ((z & 31) << 5); + } + + private static int getTimestamp() { + return (int) (System.currentTimeMillis() / 1000L); + } + + public void flush() throws IOException { + flushWrapper(); // sync + } + + public void flushWrapper() { + try { + save(); + } catch (IOException e) { + LOGGER.error("Failed to flush region file {}", path.toAbsolutePath(), e); + } + } + + public boolean doesChunkExist(ChunkPos pos) throws Exception { + throw new Exception("doesChunkExist is a stub"); + } + + private synchronized void save() throws IOException { + long timestamp = getTimestamp(); + short chunkCount = 0; + + File tempFile = new File(path.toString() + ".tmp"); + + try (FileOutputStream fileStream = new FileOutputStream(tempFile); + ByteArrayOutputStream zstdByteArray = new ByteArrayOutputStream(); + ZstdOutputStream zstdStream = new ZstdOutputStream(zstdByteArray, this.compressionLevel); + DataOutputStream zstdDataStream = new DataOutputStream(zstdStream); + DataOutputStream dataStream = new DataOutputStream(fileStream)) { + + dataStream.writeLong(SUPERBLOCK); + dataStream.writeByte(VERSION); + dataStream.writeLong(timestamp); + dataStream.writeByte(this.compressionLevel); + + ArrayList byteBuffers = new ArrayList<>(); + for (int i = 0; i < 1024; i++) { + if (this.bufferUncompressedSize[i] != 0) { + chunkCount += 1; + byte[] content = new byte[bufferUncompressedSize[i]]; + this.decompressor.decompress(buffer[i], 0, content, 0, bufferUncompressedSize[i]); + + byteBuffers.add(content); + } else byteBuffers.add(null); + } + for (int i = 0; i < 1024; i++) { + zstdDataStream.writeInt(this.bufferUncompressedSize[i]); // Write uncompressed size + zstdDataStream.writeInt(this.chunkTimestamps[i]); // Write timestamp + } + for (int i = 0; i < 1024; i++) { + if (byteBuffers.get(i) != null) + zstdDataStream.write(byteBuffers.get(i), 0, byteBuffers.get(i).length); + } + zstdDataStream.close(); + + dataStream.writeShort(chunkCount); + + byte[] compressed = zstdByteArray.toByteArray(); + + dataStream.writeInt(compressed.length); + dataStream.writeLong(0); + + dataStream.write(compressed, 0, compressed.length); + dataStream.writeLong(SUPERBLOCK); + + dataStream.flush(); + fileStream.getFD().sync(); + fileStream.getChannel().force(true); // Ensure atomicity on Btrfs + } + Files.move(tempFile.toPath(), this.path, StandardCopyOption.REPLACE_EXISTING); + this.lastFlushed = System.nanoTime(); + } + + public synchronized void write(ChunkPos pos, ByteBuffer buffer) { + try { + byte[] b = toByteArray(new ByteArrayInputStream(buffer.array())); + int uncompressedSize = b.length; + + int maxCompressedLength = this.compressor.maxCompressedLength(b.length); + byte[] compressed = new byte[maxCompressedLength]; + int compressedLength = this.compressor.compress(b, 0, b.length, compressed, 0, maxCompressedLength); + b = new byte[compressedLength]; + System.arraycopy(compressed, 0, b, 0, compressedLength); + + int index = getChunkIndex(pos.x, pos.z); + this.buffer[index] = b; + this.chunkTimestamps[index] = getTimestamp(); + this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] = uncompressedSize; + } catch (IOException e) { + LOGGER.error("Chunk write IOException {} {}", e, this.path); + } + + if ((System.nanoTime() - this.lastFlushed) >= TimeUnit.NANOSECONDS.toSeconds(DivineConfig.linearFlushFrequency)) { + this.flushWrapper(); + } + } + + public DataOutputStream getChunkDataOutputStream(ChunkPos pos) { + return new DataOutputStream(new BufferedOutputStream(new ChunkBuffer(pos))); + } + + private byte[] toByteArray(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] tempBuffer = new byte[4096]; + + int length; + while ((length = in.read(tempBuffer)) >= 0) { + out.write(tempBuffer, 0, length); + } + + return out.toByteArray(); + } + + @Nullable + public synchronized DataInputStream getChunkDataInputStream(ChunkPos pos) { + if (this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] != 0) { + byte[] content = new byte[bufferUncompressedSize[getChunkIndex(pos.x, pos.z)]]; + this.decompressor.decompress(this.buffer[getChunkIndex(pos.x, pos.z)], 0, content, 0, bufferUncompressedSize[getChunkIndex(pos.x, pos.z)]); + return new DataInputStream(new ByteArrayInputStream(content)); + } + return null; + } + + public void clear(ChunkPos pos) { + int i = getChunkIndex(pos.x, pos.z); + this.buffer[i] = null; + this.bufferUncompressedSize[i] = 0; + this.chunkTimestamps[i] = getTimestamp(); + this.flushWrapper(); + } + + public Path getPath() { + return this.path; + } + + public boolean hasChunk(ChunkPos pos) { + return this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] > 0; + } + + public void close() throws IOException { + if (closed) return; + closed = true; + flush(); // sync + } + + public boolean recalculateHeader() { + return false; + } + + public void setOversized(int x, int z, boolean something) { + } + + public CompoundTag getOversizedData(int x, int z) throws IOException { + throw new IOException("getOversizedData is a stub " + this.path); + } + + public boolean isOversized(int x, int z) { + return false; + } + + private class ChunkBuffer extends ByteArrayOutputStream { + private final ChunkPos pos; + + public ChunkBuffer(ChunkPos chunkcoordintpair) { + super(); + this.pos = chunkcoordintpair; + } + + public void close() { + ByteBuffer bytebuffer = ByteBuffer.wrap(this.buf, 0, this.count); + LinearRegionFile.this.write(this.pos, bytebuffer); + } + } +} diff --git a/divinemc-server/src/main/java/space/bxteam/divinemc/region/RegionFileFormat.java b/divinemc-server/src/main/java/space/bxteam/divinemc/region/RegionFileFormat.java new file mode 100644 index 0000000..a799c9b --- /dev/null +++ b/divinemc-server/src/main/java/space/bxteam/divinemc/region/RegionFileFormat.java @@ -0,0 +1,16 @@ +package space.bxteam.divinemc.region; + +public enum RegionFileFormat { + ANVIL, + LINEAR, + INVALID; + + public static RegionFileFormat fromString(String format) { + for (RegionFileFormat rff : values()) { + if (rff.name().equalsIgnoreCase(format)) { + return rff; + } + } + return RegionFileFormat.INVALID; + } +}