From 4e6ac403905ca465dc73bf04801b230da043af85 Mon Sep 17 00:00:00 2001 From: NONPLAYT <76615486+NONPLAYT@users.noreply.github.com> Date: Sun, 16 Mar 2025 21:43:16 +0300 Subject: [PATCH] implement back linear region format --- build-data/divinemc.at | 5 + build.gradle.kts | 6 + divinemc-server/build.gradle.kts.patch | 10 +- .../0044-Linear-region-file-format.patch | 328 ++++++++++++++++++ .../org/bxteam/divinemc/DivineConfig.java | 32 ++ .../region/EnumRegionFileExtension.java | 55 +++ .../linearpaper/region/IRegionFile.java | 39 +++ .../region/IRegionFileFactory.java | 48 +++ .../linearpaper/region/LinearRegionFile.java | 308 ++++++++++++++++ 9 files changed, 828 insertions(+), 3 deletions(-) create mode 100644 divinemc-server/minecraft-patches/features/0044-Linear-region-file-format.patch create mode 100644 divinemc-server/src/main/java/org/stupidcraft/linearpaper/region/EnumRegionFileExtension.java create mode 100644 divinemc-server/src/main/java/org/stupidcraft/linearpaper/region/IRegionFile.java create mode 100644 divinemc-server/src/main/java/org/stupidcraft/linearpaper/region/IRegionFileFactory.java create mode 100644 divinemc-server/src/main/java/org/stupidcraft/linearpaper/region/LinearRegionFile.java diff --git a/build-data/divinemc.at b/build-data/divinemc.at index 2b0ec73..866c347 100644 --- a/build-data/divinemc.at +++ b/build-data/divinemc.at @@ -23,6 +23,11 @@ public net.minecraft.world.level.chunk.LevelChunkSection tickingFluidCount public net.minecraft.world.level.chunk.PalettedContainer palette public net.minecraft.world.level.chunk.PalettedContainer strategy public net.minecraft.world.level.chunk.PalettedContainer$Data palette +public net.minecraft.world.level.chunk.storage.RegionFile getOversizedData(II)Lnet/minecraft/nbt/CompoundTag; +public net.minecraft.world.level.chunk.storage.RegionFile isOversized(II)Z +public net.minecraft.world.level.chunk.storage.RegionFile recalculateHeader()Z +public net.minecraft.world.level.chunk.storage.RegionFile setOversized(IIZ)V +public net.minecraft.world.level.chunk.storage.RegionFile write(Lnet/minecraft/world/level/ChunkPos;Ljava/nio/ByteBuffer;)V public net.minecraft.world.level.levelgen.DensityFunctions$BlendAlpha public net.minecraft.world.level.levelgen.DensityFunctions$BlendDensity public net.minecraft.world.level.levelgen.DensityFunctions$BlendOffset diff --git a/build.gradle.kts b/build.gradle.kts index 92d2a2a..ec8268b 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -102,3 +102,9 @@ subprojects { } } } + +tasks.register("printMinecraftVersion") { + doLast { + println(providers.gradleProperty("mcVersion").get().trim()) + } +} diff --git a/divinemc-server/build.gradle.kts.patch b/divinemc-server/build.gradle.kts.patch index 4aec0fd..b33b5cb 100644 --- a/divinemc-server/build.gradle.kts.patch +++ b/divinemc-server/build.gradle.kts.patch @@ -57,7 +57,7 @@ implementation("ca.spottedleaf:concurrentutil:0.0.3") implementation("org.jline:jline-terminal-ffm:3.27.1") // use ffm on java 22+ implementation("org.jline:jline-terminal-jni:3.27.1") // fall back to jni on java 21 -@@ -177,6 +_,15 @@ +@@ -177,6 +_,17 @@ implementation("org.mozilla:rhino-engine:1.7.14") // Purpur implementation("dev.omega24:upnp4j:1.0") // Purpur @@ -68,6 +68,8 @@ + } + implementation("net.objecthunter:exp4j:0.4.8") + implementation("org.agrona:agrona:2.0.1") ++ implementation("com.github.luben:zstd-jni:1.5.6-9") ++ implementation("org.lz4:lz4-java:1.8.0") + // DivineMC end + runtimeOnly("org.apache.maven:maven-resolver-provider:3.9.6") @@ -110,13 +112,15 @@ - "Specification-Vendor" to "Purpur Team", // Purpur - "Brand-Id" to "purpurmc:purpur", // Purpur - "Brand-Name" to "Purpur", // Purpur +- "Build-Number" to (build ?: ""), + "Specification-Vendor" to "BX Team", // DivineMC + "Brand-Id" to "bxteam:divinemc", // DivineMC + "Brand-Name" to "DivineMC", // DivineMC - "Build-Number" to (build ?: ""), ++ "Build-Number" to "456", "Build-Time" to buildTime.toString(), "Git-Branch" to gitBranch, - "Git-Commit" to gitHash, +- "Git-Commit" to gitHash, ++ "Git-Commit" to "4176f4d", + "Experimental" to experimental, // DivineMC - Experimental flag ) for (tld in setOf("net", "com", "org")) { diff --git a/divinemc-server/minecraft-patches/features/0044-Linear-region-file-format.patch b/divinemc-server/minecraft-patches/features/0044-Linear-region-file-format.patch new file mode 100644 index 0000000..eef930b --- /dev/null +++ b/divinemc-server/minecraft-patches/features/0044-Linear-region-file-format.patch @@ -0,0 +1,328 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: NONPLAYT <76615486+NONPLAYT@users.noreply.github.com> +Date: Sun, 16 Mar 2025 21:16:55 +0300 +Subject: [PATCH] Linear region file format + + +diff --git a/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java b/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java +index a814512fcfb85312474ae2c2c21443843bf57831..c21e04c1d43797db221e4712fcc987a44eaa983c 100644 +--- a/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java ++++ b/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java +@@ -8,9 +8,9 @@ public interface ChunkSystemRegionFileStorage { + + public boolean moonrise$doesRegionFileNotExistNoIO(final int chunkX, final int chunkZ); + +- public RegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ); ++ public org.stupidcraft.linearpaper.region.IRegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ); // DivineMC - Linear region file format + +- public RegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException; ++ public org.stupidcraft.linearpaper.region.IRegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException; // DivineMC - Linear region file format + + public MoonriseRegionFileIO.RegionDataController.WriteData moonrise$startWrite( + final int chunkX, final int chunkZ, final CompoundTag compound +diff --git a/ca/spottedleaf/moonrise/patches/chunk_system/io/MoonriseRegionFileIO.java b/ca/spottedleaf/moonrise/patches/chunk_system/io/MoonriseRegionFileIO.java +index 98fbc5c8044bd945d64569f13412a6e7e49a4e7f..57ae23fdd5a95e7670eb4dfaca0d290edfc1d25c 100644 +--- a/ca/spottedleaf/moonrise/patches/chunk_system/io/MoonriseRegionFileIO.java ++++ b/ca/spottedleaf/moonrise/patches/chunk_system/io/MoonriseRegionFileIO.java +@@ -1260,7 +1260,7 @@ public final class MoonriseRegionFileIO { + this.regionDataController.finishWrite(this.chunkX, this.chunkZ, writeData); + // Paper start - flush regionfiles on save + if (this.world.paperConfig().chunks.flushRegionsOnSave) { +- final RegionFile regionFile = this.regionDataController.getCache().moonrise$getRegionFileIfLoaded(this.chunkX, this.chunkZ); ++ final org.stupidcraft.linearpaper.region.IRegionFile regionFile = this.regionDataController.getCache().moonrise$getRegionFileIfLoaded(this.chunkX, this.chunkZ); // DivineMC - Linear region file format + if (regionFile != null) { + regionFile.flush(); + } // else: evicted from cache, which should have called flush +@@ -1470,7 +1470,7 @@ public final class MoonriseRegionFileIO { + + public static interface IORunnable { + +- public void run(final RegionFile regionFile) throws IOException; ++ public void run(final org.stupidcraft.linearpaper.region.IRegionFile regionFile) throws IOException; // DivineMC - Linear region file format + + } + } +diff --git a/ca/spottedleaf/moonrise/patches/chunk_system/storage/ChunkSystemChunkBuffer.java b/ca/spottedleaf/moonrise/patches/chunk_system/storage/ChunkSystemChunkBuffer.java +index 51c126735ace8fdde89ad97b5cab62f244212db0..c466ce5c669e4b5ed835b13584a5ae5afe08fa11 100644 +--- a/ca/spottedleaf/moonrise/patches/chunk_system/storage/ChunkSystemChunkBuffer.java ++++ b/ca/spottedleaf/moonrise/patches/chunk_system/storage/ChunkSystemChunkBuffer.java +@@ -8,5 +8,5 @@ public interface ChunkSystemChunkBuffer { + + public void moonrise$setWriteOnClose(final boolean value); + +- public void moonrise$write(final RegionFile regionFile) throws IOException; ++ public void moonrise$write(final org.stupidcraft.linearpaper.region.IRegionFile regionFile) throws IOException; // DivineMC - Linear region file format + } +diff --git a/net/minecraft/server/MinecraftServer.java b/net/minecraft/server/MinecraftServer.java +index 3bb5f956c2763007a83bc209d339c0159cb1ea8e..466fc421e0191ce1854e114fd3a013d241bc89f3 100644 +--- a/net/minecraft/server/MinecraftServer.java ++++ b/net/minecraft/server/MinecraftServer.java +@@ -965,10 +965,10 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop> progressMap = Reference2FloatMaps.synchronize(new Reference2FloatOpenHashMap<>()); + volatile Component status = Component.translatable("optimizeWorld.stage.counting"); +- static final Pattern REGEX = Pattern.compile("^r\\.(-?[0-9]+)\\.(-?[0-9]+)\\.mca$"); ++ static final Pattern REGEX = Pattern.compile("^r\\.(-?[0-9]+)\\.(-?[0-9]+)\\.(linear | mca)$"); // DivineMC - Linear region file format + final DimensionDataStorage overworldDataStorage; + + public WorldUpgrader( +@@ -261,7 +261,7 @@ public class WorldUpgrader implements AutoCloseable { + } + + private static List getAllChunkPositions(RegionStorageInfo regionStorageInfo, Path path) { +- File[] files = path.toFile().listFiles((directory, filename) -> filename.endsWith(".mca")); ++ File[] files = path.toFile().listFiles((directory, filename) -> filename.endsWith(".linear") || filename.endsWith(".mca")); // DivineMC - Linear region file format + if (files == null) { + return List.of(); + } else { +@@ -274,7 +274,7 @@ public class WorldUpgrader implements AutoCloseable { + int i1 = Integer.parseInt(matcher.group(2)) << 5; + List list1 = Lists.newArrayList(); + +- try (RegionFile regionFile = new RegionFile(regionStorageInfo, file.toPath(), path, true)) { ++ try (org.stupidcraft.linearpaper.region.IRegionFile regionFile = org.stupidcraft.linearpaper.region.IRegionFileFactory.getAbstractRegionFile(regionStorageInfo, file.toPath(), path, true)) { // DivineMC - Linear region file format + for (int i2 = 0; i2 < 32; i2++) { + for (int i3 = 0; i3 < 32; i3++) { + ChunkPos chunkPos = new ChunkPos(i2 + i, i3 + i1); +@@ -322,7 +322,7 @@ public class WorldUpgrader implements AutoCloseable { + + protected abstract boolean tryProcessOnePosition(T chunkStorage, ChunkPos chunkPos, ResourceKey dimension); + +- private void onFileFinished(RegionFile regionFile) { ++ private void onFileFinished(org.stupidcraft.linearpaper.region.IRegionFile regionFile) { // DivineMC - Linear region file format + if (WorldUpgrader.this.recreateRegionFiles) { + if (this.previousWriteFuture != null) { + this.previousWriteFuture.join(); +@@ -424,7 +424,7 @@ public class WorldUpgrader implements AutoCloseable { + } + } + +- record FileToUpgrade(RegionFile file, List chunksToUpgrade) { ++ record FileToUpgrade(org.stupidcraft.linearpaper.region.IRegionFile file, List chunksToUpgrade) { // DivineMC - Linear region file format + } + + class PoiUpgrader extends WorldUpgrader.SimpleRegionStorageUpgrader { +diff --git a/net/minecraft/world/level/chunk/storage/RegionFile.java b/net/minecraft/world/level/chunk/storage/RegionFile.java +index c72494e757a9dc50e053dbc873f7b30e83d5cb8c..5d29e3046485dcdda9356f6bfe6f6463e9b9bef4 100644 +--- a/net/minecraft/world/level/chunk/storage/RegionFile.java ++++ b/net/minecraft/world/level/chunk/storage/RegionFile.java +@@ -22,7 +22,7 @@ import net.minecraft.util.profiling.jfr.JvmProfiler; + import net.minecraft.world.level.ChunkPos; + import org.slf4j.Logger; + +-public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patches.chunk_system.storage.ChunkSystemRegionFile { // Paper - rewrite chunk system ++public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patches.chunk_system.storage.ChunkSystemRegionFile, org.stupidcraft.linearpaper.region.IRegionFile { // Paper - rewrite chunk system // DivineMC - Linear region file format + private static final Logger LOGGER = LogUtils.getLogger(); + public static final int MAX_CHUNK_SIZE = 500 * 1024 * 1024; // Paper - don't write garbage data to disk if writing serialization fails + private static final int SECTOR_BYTES = 4096; +@@ -904,7 +904,7 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche + } + + @Override +- public final void moonrise$write(final RegionFile regionFile) throws IOException { ++ public final void moonrise$write(final org.stupidcraft.linearpaper.region.IRegionFile regionFile) throws IOException { // DivineMC - Linear region file format + regionFile.write(this.pos, ByteBuffer.wrap(this.buf, 0, this.count)); + } + // Paper end - rewrite chunk system +diff --git a/net/minecraft/world/level/chunk/storage/RegionFileStorage.java b/net/minecraft/world/level/chunk/storage/RegionFileStorage.java +index 16cd10ab8de69ca3d29c84cf93715645322fd72a..1061d1480af0d4947d0f7f1b0028b4196cf25588 100644 +--- a/net/minecraft/world/level/chunk/storage/RegionFileStorage.java ++++ b/net/minecraft/world/level/chunk/storage/RegionFileStorage.java +@@ -18,7 +18,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + private static final org.slf4j.Logger LOGGER = com.mojang.logging.LogUtils.getLogger(); // Paper + public static final String ANVIL_EXTENSION = ".mca"; + private static final int MAX_CACHE_SIZE = 256; +- public final Long2ObjectLinkedOpenHashMap regionCache = new Long2ObjectLinkedOpenHashMap<>(); ++ public final Long2ObjectLinkedOpenHashMap regionCache = new Long2ObjectLinkedOpenHashMap<>(); // DivineMC - Linear region file format + private final RegionStorageInfo info; + private final Path folder; + private final boolean sync; +@@ -33,7 +33,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + @Nullable + public static ChunkPos getRegionFileCoordinates(Path file) { + String fileName = file.getFileName().toString(); +- if (!fileName.startsWith("r.") || !fileName.endsWith(".mca")) { ++ if (!fileName.startsWith("r.") || !fileName.endsWith(".mca") || !fileName.endsWith(".linear")) { // DivineMC - Linear region file format + return null; + } + +@@ -58,6 +58,12 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + private static final int MAX_NON_EXISTING_CACHE = 1024 * 4; + private final it.unimi.dsi.fastutil.longs.LongLinkedOpenHashSet nonExistingRegionFiles = new it.unimi.dsi.fastutil.longs.LongLinkedOpenHashSet(); + private static String getRegionFileName(final int chunkX, final int chunkZ) { ++ // DivineMC start - Linear region file format ++ if (org.bxteam.divinemc.DivineConfig.regionFormatTypeName == org.stupidcraft.linearpaper.region.EnumRegionFileExtension.LINEAR) { ++ return "r." + (chunkX >> REGION_SHIFT) + "." + (chunkZ >> REGION_SHIFT) + ".linear"; ++ } ++ // DivineMC end - Linear region file format ++ + return "r." + (chunkX >> REGION_SHIFT) + "." + (chunkZ >> REGION_SHIFT) + ".mca"; + } + +@@ -93,15 +99,15 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + } + + @Override +- public synchronized final RegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ) { ++ public synchronized final org.stupidcraft.linearpaper.region.IRegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ) { // DivineMC - Linear region file format + return this.regionCache.getAndMoveToFirst(ChunkPos.asLong(chunkX >> REGION_SHIFT, chunkZ >> REGION_SHIFT)); + } + + @Override +- public synchronized final RegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException { ++ public synchronized final org.stupidcraft.linearpaper.region.IRegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException { // DivineMC - Linear region file format + final long key = ChunkPos.asLong(chunkX >> REGION_SHIFT, chunkZ >> REGION_SHIFT); + +- RegionFile ret = this.regionCache.getAndMoveToFirst(key); ++ org.stupidcraft.linearpaper.region.IRegionFile ret = this.regionCache.getAndMoveToFirst(key); // DivineMC - Linear region file format + if (ret != null) { + return ret; + } +@@ -125,7 +131,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + + FileUtil.createDirectoriesSafe(this.folder); + +- ret = new RegionFile(this.info, regionPath, this.folder, this.sync); ++ ret = org.stupidcraft.linearpaper.region.IRegionFileFactory.getAbstractRegionFile(this.info, regionPath, this.folder, this.sync); // DivineMC - Linear region file format + + this.regionCache.putAndMoveToFirst(key, ret); + +@@ -144,11 +150,11 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + } + + final ChunkPos pos = new ChunkPos(chunkX, chunkZ); +- final RegionFile regionFile = this.getRegionFile(pos); ++ final org.stupidcraft.linearpaper.region.IRegionFile regionFile = this.getRegionFile(pos); // DivineMC - Linear region file format + + // note: not required to keep regionfile loaded after this call, as the write param takes a regionfile as input + // (and, the regionfile parameter is unused for writing until the write call) +- final ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO.RegionDataController.WriteData writeData = ((ca.spottedleaf.moonrise.patches.chunk_system.storage.ChunkSystemRegionFile)regionFile).moonrise$startWrite(compound, pos); ++ final ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO.RegionDataController.WriteData writeData = regionFile.moonrise$startWrite(compound, pos); // DivineMC - Linear region file format + + try { // Paper - implement RegionFileSizeException + try { +@@ -178,7 +184,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + ) throws IOException { + final ChunkPos pos = new ChunkPos(chunkX, chunkZ); + if (writeData.result() == ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO.RegionDataController.WriteData.WriteResult.DELETE) { +- final RegionFile regionFile = this.moonrise$getRegionFileIfExists(chunkX, chunkZ); ++ final org.stupidcraft.linearpaper.region.IRegionFile regionFile = this.moonrise$getRegionFileIfExists(chunkX, chunkZ); // DivineMC - Linear region file format + if (regionFile != null) { + regionFile.clear(pos); + } // else: didn't exist +@@ -193,7 +199,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + public final ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO.RegionDataController.ReadData moonrise$readData( + final int chunkX, final int chunkZ + ) throws IOException { +- final RegionFile regionFile = this.moonrise$getRegionFileIfExists(chunkX, chunkZ); ++ final org.stupidcraft.linearpaper.region.IRegionFile regionFile = this.moonrise$getRegionFileIfExists(chunkX, chunkZ); // DivineMC - Linear region file format + + final DataInputStream input = regionFile == null ? null : regionFile.getChunkDataInputStream(new ChunkPos(chunkX, chunkZ)); + +@@ -237,7 +243,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + } + // Paper end - rewrite chunk system + // Paper start - rewrite chunk system +- public RegionFile getRegionFile(ChunkPos chunkcoordintpair) throws IOException { ++ public org.stupidcraft.linearpaper.region.IRegionFile getRegionFile(ChunkPos chunkcoordintpair) throws IOException { // DivineMC - Linear region file format + return this.getRegionFile(chunkcoordintpair, false); + } + // Paper end - rewrite chunk system +@@ -249,7 +255,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + this.isChunkData = isChunkDataFolder(this.folder); // Paper - recalculate region file headers + } + +- @org.jetbrains.annotations.Contract("_, false -> !null") @Nullable private RegionFile getRegionFile(ChunkPos chunkPos, boolean existingOnly) throws IOException { // CraftBukkit ++ @org.jetbrains.annotations.Contract("_, false -> !null") @Nullable private org.stupidcraft.linearpaper.region.IRegionFile getRegionFile(ChunkPos chunkPos, boolean existingOnly) throws IOException { // CraftBukkit // DivineMC - Linear region file format + // Paper start - rewrite chunk system + if (existingOnly) { + return this.moonrise$getRegionFileIfExists(chunkPos.x, chunkPos.z); +@@ -257,7 +263,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + synchronized (this) { + final long key = ChunkPos.asLong(chunkPos.x >> REGION_SHIFT, chunkPos.z >> REGION_SHIFT); + +- RegionFile ret = this.regionCache.getAndMoveToFirst(key); ++ org.stupidcraft.linearpaper.region.IRegionFile ret = this.regionCache.getAndMoveToFirst(key); // DivineMC - Linear region file format + if (ret != null) { + return ret; + } +@@ -272,7 +278,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + + FileUtil.createDirectoriesSafe(this.folder); + +- ret = new RegionFile(this.info, regionPath, this.folder, this.sync); ++ ret = org.stupidcraft.linearpaper.region.IRegionFileFactory.getAbstractRegionFile(this.info, regionPath, this.folder, this.sync); // DivineMC - Linear region file format + + this.regionCache.putAndMoveToFirst(key, ret); + +@@ -286,7 +292,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + org.apache.logging.log4j.LogManager.getLogger().fatal(msg + " (" + file.toString().replaceAll(".+[\\\\/]", "") + " - " + x + "," + z + ") Go clean it up to remove this message. /minecraft:tp " + (x<<4)+" 128 "+(z<<4) + " - DO NOT REPORT THIS TO DIVINEMC - You may ask for help on Discord, but do not file an issue. These error messages can not be removed."); // DivineMC - Rebrand + } + +- private static CompoundTag readOversizedChunk(RegionFile regionfile, ChunkPos chunkCoordinate) throws IOException { ++ private static CompoundTag readOversizedChunk(org.stupidcraft.linearpaper.region.IRegionFile regionfile, ChunkPos chunkCoordinate) throws IOException { // DivineMC - Linear region file format + synchronized (regionfile) { + try (DataInputStream datainputstream = regionfile.getChunkDataInputStream(chunkCoordinate)) { + CompoundTag oversizedData = regionfile.getOversizedData(chunkCoordinate.x, chunkCoordinate.z); +@@ -321,7 +327,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + @Nullable + public CompoundTag read(ChunkPos chunkPos) throws IOException { + // CraftBukkit start - SPIGOT-5680: There's no good reason to preemptively create files on read, save that for writing +- RegionFile regionFile = this.getRegionFile(chunkPos, true); ++ org.stupidcraft.linearpaper.region.IRegionFile regionFile = this.getRegionFile(chunkPos, true); // DivineMC - Linear region file format + if (regionFile == null) { + return null; + } +@@ -360,7 +366,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + + public void scanChunk(ChunkPos chunkPos, StreamTagVisitor visitor) throws IOException { + // CraftBukkit start - SPIGOT-5680: There's no good reason to preemptively create files on read, save that for writing +- RegionFile regionFile = this.getRegionFile(chunkPos, true); ++ org.stupidcraft.linearpaper.region.IRegionFile regionFile = this.getRegionFile(chunkPos, true); // DivineMC - Linear region file format + if (regionFile == null) { + return; + } +@@ -374,7 +380,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + } + + public void write(ChunkPos chunkPos, @Nullable CompoundTag chunkData) throws IOException { // Paper - rewrite chunk system - public +- RegionFile regionFile = this.getRegionFile(chunkPos, chunkData == null); // CraftBukkit // Paper - rewrite chunk system ++ org.stupidcraft.linearpaper.region.IRegionFile regionFile = this.getRegionFile(chunkPos, chunkData == null); // CraftBukkit // Paper - rewrite chunk system // DivineMC - Linear region file format + // Paper start - rewrite chunk system + if (regionFile == null) { + // if the RegionFile doesn't exist, no point in deleting from it +@@ -404,7 +410,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + // Paper start - rewrite chunk system + synchronized (this) { + final ExceptionCollector exceptionCollector = new ExceptionCollector<>(); +- for (final RegionFile regionFile : this.regionCache.values()) { ++ for (final org.stupidcraft.linearpaper.region.IRegionFile regionFile : this.regionCache.values()) { // DivineMC - Linear region file format + try { + regionFile.close(); + } catch (final IOException ex) { +@@ -420,7 +426,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + // Paper start - rewrite chunk system + synchronized (this) { + final ExceptionCollector exceptionCollector = new ExceptionCollector<>(); +- for (final RegionFile regionFile : this.regionCache.values()) { ++ for (final org.stupidcraft.linearpaper.region.IRegionFile regionFile : this.regionCache.values()) { // DivineMC - Linear region file format + try { + regionFile.flush(); + } catch (final IOException ex) { diff --git a/divinemc-server/src/main/java/org/bxteam/divinemc/DivineConfig.java b/divinemc-server/src/main/java/org/bxteam/divinemc/DivineConfig.java index bf31252..9b3d578 100644 --- a/divinemc-server/src/main/java/org/bxteam/divinemc/DivineConfig.java +++ b/divinemc-server/src/main/java/org/bxteam/divinemc/DivineConfig.java @@ -12,6 +12,7 @@ import org.jetbrains.annotations.Nullable; import org.simpleyaml.configuration.comments.CommentType; import org.simpleyaml.configuration.file.YamlFile; import org.simpleyaml.exceptions.InvalidConfigurationException; +import org.stupidcraft.linearpaper.region.EnumRegionFileExtension; import java.io.File; import java.io.IOException; @@ -484,4 +485,35 @@ public class DivineConfig { if (asyncEntityTrackerQueueSize <= 0) asyncEntityTrackerQueueSize = asyncEntityTrackerMaxThreads * 384; } + + public static EnumRegionFileExtension regionFormatTypeName = EnumRegionFileExtension.MCA; + public static int linearCompressionLevel = 1; + public static int linearFlushFrequency = 5; + private static void linearRegionFormat() { + regionFormatTypeName = EnumRegionFileExtension.fromName(getString("settings.linear-region-format.type", regionFormatTypeName.name(), + "The type of region file format to use for storing chunk data.", + "Valid values:", + " - LINEAR: Linear region file format", + " - MCA: Anvil region file format (default)")); + linearCompressionLevel = getInt("settings.linear-region-format.compression-level", linearCompressionLevel, + "The compression level to use for the linear region file format."); + linearFlushFrequency = getInt("settings.linear-region-format.flush-frequency", linearFlushFrequency, + "The frequency in seconds to flush the linear region file format."); + + setComment("settings.linear-region-format", + "The linear region file format is a custom region file format that is designed to be more efficient than the MCA format.", + "It uses uses ZSTD compression instead of ZLIB. This format saves about 50% of disk space.", + "Read more information about linear region format at https://github.com/xymb-endcrystalme/LinearRegionFileFormatTools", + "WARNING: If you are want to use this format, make sure to create backup of your world before switching to it, there is potential risk to lose chunk data."); + + if (regionFormatTypeName == EnumRegionFileExtension.UNKNOWN) { + LOGGER.error("Unknown region file type: {}, falling back to MCA format.", regionFormatTypeName); + regionFormatTypeName = EnumRegionFileExtension.MCA; + } + + if (linearCompressionLevel > 23 || linearCompressionLevel < 1) { + LOGGER.warn("Invalid linear compression level: {}, resetting to default (1)", playerNearChunkDetectionRange); + linearCompressionLevel = 1; + } + } } diff --git a/divinemc-server/src/main/java/org/stupidcraft/linearpaper/region/EnumRegionFileExtension.java b/divinemc-server/src/main/java/org/stupidcraft/linearpaper/region/EnumRegionFileExtension.java new file mode 100644 index 0000000..2440bd3 --- /dev/null +++ b/divinemc-server/src/main/java/org/stupidcraft/linearpaper/region/EnumRegionFileExtension.java @@ -0,0 +1,55 @@ +package org.stupidcraft.linearpaper.region; + +import org.jetbrains.annotations.Contract; +import org.jetbrains.annotations.NotNull; + +import java.util.Locale; + +public enum EnumRegionFileExtension { + LINEAR(".linear"), + MCA(".mca"), + UNKNOWN(null); + + private final String extensionName; + + EnumRegionFileExtension(String extensionName) { + this.extensionName = extensionName; + } + + public String getExtensionName() { + return this.extensionName; + } + + @Contract(pure = true) + public static EnumRegionFileExtension fromName(@NotNull String name) { + switch (name.toUpperCase(Locale.ROOT)) { + case "MCA" -> { + return MCA; + } + + case "LINEAR" -> { + return LINEAR; + } + default -> { + return UNKNOWN; + } + } + } + + @Contract(pure = true) + public static EnumRegionFileExtension fromExtension(@NotNull String name) { + switch (name.toLowerCase()) { + case "mca" -> { + return MCA; + } + + case "linear" -> { + return LINEAR; + } + + default -> { + return UNKNOWN; + } + } + } +} diff --git a/divinemc-server/src/main/java/org/stupidcraft/linearpaper/region/IRegionFile.java b/divinemc-server/src/main/java/org/stupidcraft/linearpaper/region/IRegionFile.java new file mode 100644 index 0000000..a09223d --- /dev/null +++ b/divinemc-server/src/main/java/org/stupidcraft/linearpaper/region/IRegionFile.java @@ -0,0 +1,39 @@ +package org.stupidcraft.linearpaper.region; + +import ca.spottedleaf.moonrise.patches.chunk_system.storage.ChunkSystemRegionFile; +import net.minecraft.nbt.CompoundTag; +import net.minecraft.world.level.ChunkPos; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Path; + +public interface IRegionFile extends AutoCloseable, ChunkSystemRegionFile { + Path getPath(); + + void flush() throws IOException; + + void clear(ChunkPos pos) throws IOException; + + void close() throws IOException; + + void setOversized(int x, int z, boolean b) throws IOException; + + void write(ChunkPos pos, ByteBuffer buffer) throws IOException; + + boolean hasChunk(ChunkPos pos); + + boolean doesChunkExist(ChunkPos pos) throws Exception; + + boolean isOversized(int x, int z); + + boolean recalculateHeader() throws IOException; + + DataOutputStream getChunkDataOutputStream(ChunkPos pos) throws IOException; + + DataInputStream getChunkDataInputStream(ChunkPos pos) throws IOException; + + CompoundTag getOversizedData(int x, int z) throws IOException; +} diff --git a/divinemc-server/src/main/java/org/stupidcraft/linearpaper/region/IRegionFileFactory.java b/divinemc-server/src/main/java/org/stupidcraft/linearpaper/region/IRegionFileFactory.java new file mode 100644 index 0000000..3252852 --- /dev/null +++ b/divinemc-server/src/main/java/org/stupidcraft/linearpaper/region/IRegionFileFactory.java @@ -0,0 +1,48 @@ +package org.stupidcraft.linearpaper.region; + +import net.minecraft.world.level.chunk.storage.RegionFile; +import net.minecraft.world.level.chunk.storage.RegionFileVersion; +import net.minecraft.world.level.chunk.storage.RegionStorageInfo; +import org.bxteam.divinemc.DivineConfig; +import org.jetbrains.annotations.Contract; +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.nio.file.Path; + +public class IRegionFileFactory { + @Contract("_, _, _, _ -> new") + public static @NotNull IRegionFile getAbstractRegionFile(RegionStorageInfo storageKey, Path directory, Path path, boolean dsync) throws IOException { + return getAbstractRegionFile(storageKey, directory, path, RegionFileVersion.getCompressionFormat(), dsync); + } + + @Contract("_, _, _, _, _ -> new") + public static @NotNull IRegionFile getAbstractRegionFile(RegionStorageInfo storageKey, Path directory, Path path, boolean dsync, boolean canRecalcHeader) throws IOException { + return getAbstractRegionFile(storageKey, directory, path, RegionFileVersion.getCompressionFormat(), dsync, canRecalcHeader); + } + + @Contract("_, _, _, _, _ -> new") + public static @NotNull IRegionFile getAbstractRegionFile(RegionStorageInfo storageKey, Path path, Path directory, RegionFileVersion compressionFormat, boolean dsync) throws IOException { + return getAbstractRegionFile(storageKey, path, directory, compressionFormat, dsync, true); + } + + @Contract("_, _, _, _, _, _ -> new") + public static @NotNull IRegionFile getAbstractRegionFile(RegionStorageInfo storageKey, @NotNull Path path, Path directory, RegionFileVersion compressionFormat, boolean dsync, boolean canRecalcHeader) throws IOException { + final String fullFileName = path.getFileName().toString(); + final String[] fullNameSplit = fullFileName.split("\\."); + final String extensionName = fullNameSplit[fullNameSplit.length - 1]; + switch (EnumRegionFileExtension.fromExtension(extensionName)) { + case UNKNOWN -> { + return new RegionFile(storageKey, path, directory, compressionFormat, dsync); + } + + case LINEAR -> { + return new LinearRegionFile(path, DivineConfig.linearCompressionLevel); + } + + default -> { + return new RegionFile(storageKey, path, directory, compressionFormat, dsync); + } + } + } +} diff --git a/divinemc-server/src/main/java/org/stupidcraft/linearpaper/region/LinearRegionFile.java b/divinemc-server/src/main/java/org/stupidcraft/linearpaper/region/LinearRegionFile.java new file mode 100644 index 0000000..2ec67ce --- /dev/null +++ b/divinemc-server/src/main/java/org/stupidcraft/linearpaper/region/LinearRegionFile.java @@ -0,0 +1,308 @@ +package org.stupidcraft.linearpaper.region; + +import ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO; +import com.github.luben.zstd.ZstdInputStream; +import com.github.luben.zstd.ZstdOutputStream; +import com.mojang.logging.LogUtils; +import net.jpountz.lz4.LZ4Compressor; +import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4FastDecompressor; +import net.minecraft.nbt.CompoundTag; +import net.minecraft.world.level.ChunkPos; +import org.bxteam.divinemc.DivineConfig; +import org.slf4j.Logger; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; + +public class LinearRegionFile implements IRegionFile { + private static final long SUPERBLOCK = -4323716122432332390L; + private static final byte VERSION = 2; + private static final int HEADER_SIZE = 32; + private static final int FOOTER_SIZE = 8; + private static final Logger LOGGER = LogUtils.getLogger(); + private static final List SUPPORTED_VERSIONS = Arrays.asList((byte) 1, (byte) 2); + private final byte[][] buffer = new byte[1024][]; + private final int[] bufferUncompressedSize = new int[1024]; + private final int[] chunkTimestamps = new int[1024]; + private final LZ4Compressor compressor; + private final LZ4FastDecompressor decompressor; + private final int compressionLevel; + public boolean closed = false; + public Path path; + private volatile long lastFlushed = System.nanoTime(); + + public LinearRegionFile(Path file, int compression) throws IOException { + this.path = file; + this.compressionLevel = compression; + this.compressor = LZ4Factory.fastestInstance().fastCompressor(); + this.decompressor = LZ4Factory.fastestInstance().fastDecompressor(); + + File regionFile = new File(this.path.toString()); + + Arrays.fill(this.bufferUncompressedSize, 0); + + if (!regionFile.canRead()) return; + + try (FileInputStream fileStream = new FileInputStream(regionFile); + DataInputStream rawDataStream = new DataInputStream(fileStream)) { + + long superBlock = rawDataStream.readLong(); + if (superBlock != SUPERBLOCK) + throw new RuntimeException("Invalid superblock: " + superBlock + " in " + file); + + byte version = rawDataStream.readByte(); + if (!SUPPORTED_VERSIONS.contains(version)) + throw new RuntimeException("Invalid version: " + version + " in " + file); + + rawDataStream.skipBytes(11); + + int dataCount = rawDataStream.readInt(); + long fileLength = file.toFile().length(); + if (fileLength != HEADER_SIZE + dataCount + FOOTER_SIZE) + throw new IOException("Invalid file length: " + this.path + " " + fileLength + " " + (HEADER_SIZE + dataCount + FOOTER_SIZE)); + + rawDataStream.skipBytes(8); + + byte[] rawCompressed = new byte[dataCount]; + rawDataStream.readFully(rawCompressed, 0, dataCount); + + superBlock = rawDataStream.readLong(); + if (superBlock != SUPERBLOCK) + throw new IOException("Footer superblock invalid " + this.path); + + try (DataInputStream dataStream = new DataInputStream(new ZstdInputStream(new ByteArrayInputStream(rawCompressed)))) { + + int[] starts = new int[1024]; + for (int i = 0; i < 1024; i++) { + starts[i] = dataStream.readInt(); + dataStream.skipBytes(4); + } + + for (int i = 0; i < 1024; i++) { + if (starts[i] > 0) { + int size = starts[i]; + byte[] b = new byte[size]; + dataStream.readFully(b, 0, size); + + int maxCompressedLength = this.compressor.maxCompressedLength(size); + byte[] compressed = new byte[maxCompressedLength]; + int compressedLength = this.compressor.compress(b, 0, size, compressed, 0, maxCompressedLength); + b = new byte[compressedLength]; + System.arraycopy(compressed, 0, b, 0, compressedLength); + + this.buffer[i] = b; + this.bufferUncompressedSize[i] = size; + } + } + } + } + } + + private static int getChunkIndex(int x, int z) { + return (x & 31) + ((z & 31) << 5); + } + + private static int getTimestamp() { + return (int) (System.currentTimeMillis() / 1000L); + } + + public void flush() throws IOException { + flushWrapper(); + } + + public void flushWrapper() { + try { + save(); + } catch (IOException e) { + LOGGER.error("Failed to flush region file {}", path.toAbsolutePath(), e); + } + } + + public boolean doesChunkExist(ChunkPos pos) throws Exception { + throw new Exception("doesChunkExist is a stub"); + } + + private synchronized void save() throws IOException { + long timestamp = getTimestamp(); + short chunkCount = 0; + + File tempFile = new File(path.toString() + ".tmp"); + + try (FileOutputStream fileStream = new FileOutputStream(tempFile); + ByteArrayOutputStream zstdByteArray = new ByteArrayOutputStream(); + ZstdOutputStream zstdStream = new ZstdOutputStream(zstdByteArray, this.compressionLevel); + DataOutputStream zstdDataStream = new DataOutputStream(zstdStream); + DataOutputStream dataStream = new DataOutputStream(fileStream)) { + + dataStream.writeLong(SUPERBLOCK); + dataStream.writeByte(VERSION); + dataStream.writeLong(timestamp); + dataStream.writeByte(this.compressionLevel); + + ArrayList byteBuffers = new ArrayList<>(); + for (int i = 0; i < 1024; i++) { + if (this.bufferUncompressedSize[i] != 0) { + chunkCount += 1; + byte[] content = new byte[bufferUncompressedSize[i]]; + this.decompressor.decompress(buffer[i], 0, content, 0, bufferUncompressedSize[i]); + + byteBuffers.add(content); + } else byteBuffers.add(null); + } + for (int i = 0; i < 1024; i++) { + zstdDataStream.writeInt(this.bufferUncompressedSize[i]); + zstdDataStream.writeInt(this.chunkTimestamps[i]); + } + for (int i = 0; i < 1024; i++) { + if (byteBuffers.get(i) != null) + zstdDataStream.write(byteBuffers.get(i), 0, byteBuffers.get(i).length); + } + zstdDataStream.close(); + + dataStream.writeShort(chunkCount); + + byte[] compressed = zstdByteArray.toByteArray(); + + dataStream.writeInt(compressed.length); + dataStream.writeLong(0); + + dataStream.write(compressed, 0, compressed.length); + dataStream.writeLong(SUPERBLOCK); + + dataStream.flush(); + fileStream.getFD().sync(); + fileStream.getChannel().force(true); + } + Files.move(tempFile.toPath(), this.path, StandardCopyOption.REPLACE_EXISTING); + this.lastFlushed = System.nanoTime(); + } + + public synchronized void write(ChunkPos pos, ByteBuffer buffer) { + try { + byte[] b = toByteArray(new ByteArrayInputStream(buffer.array())); + int uncompressedSize = b.length; + + int maxCompressedLength = this.compressor.maxCompressedLength(b.length); + byte[] compressed = new byte[maxCompressedLength]; + int compressedLength = this.compressor.compress(b, 0, b.length, compressed, 0, maxCompressedLength); + b = new byte[compressedLength]; + System.arraycopy(compressed, 0, b, 0, compressedLength); + + int index = getChunkIndex(pos.x, pos.z); + this.buffer[index] = b; + this.chunkTimestamps[index] = getTimestamp(); + this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] = uncompressedSize; + } catch (IOException e) { + LOGGER.error("Chunk write IOException {} {}", e, this.path); + } + + if ((System.nanoTime() - this.lastFlushed) >= TimeUnit.NANOSECONDS.toSeconds(DivineConfig.linearFlushFrequency)) { + this.flushWrapper(); + } + } + + public DataOutputStream getChunkDataOutputStream(ChunkPos pos) { + return new DataOutputStream(new BufferedOutputStream(new ChunkBuffer(pos))); + } + + @Override + public MoonriseRegionFileIO.RegionDataController.WriteData moonrise$startWrite(CompoundTag data, ChunkPos pos) { + final DataOutputStream out = this.getChunkDataOutputStream(pos); + + return new ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO.RegionDataController.WriteData( + data, ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO.RegionDataController.WriteData.WriteResult.WRITE, + out, regionFile -> out.close() + ); + } + + private byte[] toByteArray(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] tempBuffer = new byte[4096]; + + int length; + while ((length = in.read(tempBuffer)) >= 0) { + out.write(tempBuffer, 0, length); + } + + return out.toByteArray(); + } + + @Nullable + public synchronized DataInputStream getChunkDataInputStream(ChunkPos pos) { + if (this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] != 0) { + byte[] content = new byte[bufferUncompressedSize[getChunkIndex(pos.x, pos.z)]]; + this.decompressor.decompress(this.buffer[getChunkIndex(pos.x, pos.z)], 0, content, 0, bufferUncompressedSize[getChunkIndex(pos.x, pos.z)]); + return new DataInputStream(new ByteArrayInputStream(content)); + } + return null; + } + + public void clear(ChunkPos pos) { + int i = getChunkIndex(pos.x, pos.z); + this.buffer[i] = null; + this.bufferUncompressedSize[i] = 0; + this.chunkTimestamps[i] = getTimestamp(); + this.flushWrapper(); + } + + public Path getPath() { + return this.path; + } + + public boolean hasChunk(ChunkPos pos) { + return this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] > 0; + } + + public void close() throws IOException { + if (closed) return; + closed = true; + flush(); + } + + public boolean recalculateHeader() { + return false; + } + + public void setOversized(int x, int z, boolean something) { + } + + public CompoundTag getOversizedData(int x, int z) throws IOException { + throw new IOException("getOversizedData is a stub " + this.path); + } + + public boolean isOversized(int x, int z) { + return false; + } + + private class ChunkBuffer extends ByteArrayOutputStream { + private final ChunkPos pos; + + public ChunkBuffer(ChunkPos chunkcoordintpair) { + super(); + this.pos = chunkcoordintpair; + } + + public void close() { + ByteBuffer bytebuffer = ByteBuffer.wrap(this.buf, 0, this.count); + LinearRegionFile.this.write(this.pos, bytebuffer); + } + } +}