From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: HaHaWTH <102713261+HaHaWTH@users.noreply.github.com> Date: Sun, 30 Jun 2024 00:35:19 +0800 Subject: [PATCH] Linear region file format Original license: MIT Original project: https://github.com/LuminolMC/Luminol Linear is a region file format that uses ZSTD compression instead of ZLIB. This format saves about 50% of disk space. Documentation: https://github.com/xymb-endcrystalme/LinearRegionFileFormatTools diff --git a/build.gradle.kts b/build.gradle.kts index 635ec3ef8bd96f73527ee50ca87c8f2e6b8232b2..92fc2ab1fd20d31d1c5476f8f2349f0f64fbec9d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -33,6 +33,11 @@ dependencies { } // Leaf end - Leaf Config + // LinearPaper start + implementation("com.github.luben:zstd-jni:1.5.6-9") + implementation("org.lz4:lz4-java:1.8.0") + // LinearPaper end + // Paper start // Leaf start - Bump Dependencies implementation("org.jline:jline-terminal-ffm:3.28.0") // use ffm on java 22+ // Leaf Bu diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java index a814512fcfb85312474ae2c2c21443843bf57831..f80c75c561313625b694b433692aa429b8f8fde9 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java @@ -8,9 +8,9 @@ public interface ChunkSystemRegionFileStorage { public boolean moonrise$doesRegionFileNotExistNoIO(final int chunkX, final int chunkZ); - public RegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ); + public org.stupidcraft.linearpaper.region.IRegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ); // LinearPaper - public RegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException; + public org.stupidcraft.linearpaper.region.IRegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException; // LinearPaper public MoonriseRegionFileIO.RegionDataController.WriteData moonrise$startWrite( final int chunkX, final int chunkZ, final CompoundTag compound diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/MoonriseRegionFileIO.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/MoonriseRegionFileIO.java index 1acea58838f057ab87efd103cbecb6f5aeaef393..48a6d8b534943393c26180fbf341b77bd2d5bc48 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/MoonriseRegionFileIO.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/MoonriseRegionFileIO.java @@ -1462,7 +1462,7 @@ public final class MoonriseRegionFileIO { public static interface IORunnable { - public void run(final RegionFile regionFile) throws IOException; + public void run(final org.stupidcraft.linearpaper.region.IRegionFile regionFile) throws IOException; // LinearPaper } } diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/storage/ChunkSystemChunkBuffer.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/storage/ChunkSystemChunkBuffer.java index 51c126735ace8fdde89ad97b5cab62f244212db0..4046f0aaa153e00277bf14f009fbe14aa8859fec 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/storage/ChunkSystemChunkBuffer.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/storage/ChunkSystemChunkBuffer.java @@ -8,5 +8,5 @@ public interface ChunkSystemChunkBuffer { public void moonrise$setWriteOnClose(final boolean value); - public void moonrise$write(final RegionFile regionFile) throws IOException; + public void moonrise$write(final org.stupidcraft.linearpaper.region.IRegionFile regionFile) throws IOException; // LinearPaper } diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java index 2d231c01fe752d3a97965a0bef01e7c0f6cb1611..87e52d6dcd855c80b3e6b4f4010b596bf8649c80 100644 --- a/src/main/java/net/minecraft/server/MinecraftServer.java +++ b/src/main/java/net/minecraft/server/MinecraftServer.java @@ -985,10 +985,10 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop> progressMap = Reference2FloatMaps.synchronize(new Reference2FloatOpenHashMap()); volatile Component status = Component.translatable("optimizeWorld.stage.counting"); - static final Pattern REGEX = Pattern.compile("^r\\.(-?[0-9]+)\\.(-?[0-9]+)\\.mca$"); + static final Pattern REGEX = Pattern.compile("^r\\.(-?[0-9]+)\\.(-?[0-9]+)\\.(linear | mca)$"); // LinearPaper final DimensionDataStorage overworldDataStorage; public WorldUpgrader(LevelStorageSource.LevelStorageAccess session, DataFixer dataFixer, RegistryAccess dynamicRegistryManager, boolean eraseCache, boolean recreateRegionFiles) { @@ -399,7 +399,7 @@ public class WorldUpgrader implements AutoCloseable { private static List getAllChunkPositions(RegionStorageInfo key, Path regionDirectory) { File[] afile = regionDirectory.toFile().listFiles((file, s) -> { - return s.endsWith(".mca"); + return s.endsWith(".linear") || s.endsWith(".mca"); // LinearPaper }); if (afile == null) { @@ -419,7 +419,7 @@ public class WorldUpgrader implements AutoCloseable { List list1 = Lists.newArrayList(); try { - RegionFile regionfile = new RegionFile(key, file.toPath(), regionDirectory, true); + org.stupidcraft.linearpaper.region.IRegionFile regionfile = org.stupidcraft.linearpaper.region.IRegionFileFactory.getAbstractRegionFile(key, file.toPath(), regionDirectory, true); // LinearPaper try { for (int i1 = 0; i1 < 32; ++i1) { @@ -482,7 +482,7 @@ public class WorldUpgrader implements AutoCloseable { protected abstract boolean tryProcessOnePosition(T storage, ChunkPos chunkPos, ResourceKey worldKey); - private void onFileFinished(RegionFile regionFile) { + private void onFileFinished(org.stupidcraft.linearpaper.region.IRegionFile regionFile) { // LinearPaper if (WorldUpgrader.this.recreateRegionFiles) { if (this.previousWriteFuture != null) { this.previousWriteFuture.join(); @@ -507,7 +507,7 @@ public class WorldUpgrader implements AutoCloseable { } } - static record FileToUpgrade(RegionFile file, List chunksToUpgrade) { + static record FileToUpgrade(org.stupidcraft.linearpaper.region.IRegionFile file, List chunksToUpgrade) { // LinearPaper } diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java index b26a5b76cd30c96ce15ced5ac51222a8333bde52..dd42303de58274c24394da26bda52bf0d09df7b2 100644 --- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java +++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java @@ -28,7 +28,7 @@ import net.minecraft.nbt.NbtIo; // Paper import net.minecraft.world.level.ChunkPos; import org.slf4j.Logger; -public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patches.chunk_system.storage.ChunkSystemRegionFile { // Paper - rewrite chunk system +public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patches.chunk_system.storage.ChunkSystemRegionFile, org.stupidcraft.linearpaper.region.IRegionFile { // Paper - rewrite chunk system // LinearPaper private static final Logger LOGGER = LogUtils.getLogger(); private static final int SECTOR_BYTES = 4096; @@ -129,7 +129,7 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche } // note: only call for CHUNK regionfiles - boolean recalculateHeader() throws IOException { + public boolean recalculateHeader() throws IOException { // LinearPaper - make public if (!this.canRecalcHeader) { return false; } @@ -810,7 +810,7 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche } } - protected synchronized void write(ChunkPos pos, ByteBuffer buf) throws IOException { + public synchronized void write(ChunkPos pos, ByteBuffer buf) throws IOException { // LinearPaper - protected -> public int i = RegionFile.getOffsetIndex(pos); int j = this.offsets.get(i); int k = RegionFile.getSectorNumber(j); @@ -952,10 +952,10 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche private static int getChunkIndex(int x, int z) { return (x & 31) + (z & 31) * 32; } - synchronized boolean isOversized(int x, int z) { + public synchronized boolean isOversized(int x, int z) { // LinearPaper - make public return this.oversized[getChunkIndex(x, z)] == 1; } - synchronized void setOversized(int x, int z, boolean oversized) throws IOException { + public synchronized void setOversized(int x, int z, boolean oversized) throws IOException { // LinearPaper - make public final int offset = getChunkIndex(x, z); boolean previous = this.oversized[offset] == 1; this.oversized[offset] = (byte) (oversized ? 1 : 0); @@ -994,7 +994,7 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche return this.path.getParent().resolve(this.path.getFileName().toString().replaceAll("\\.mca$", "") + "_oversized_" + x + "_" + z + ".nbt"); } - synchronized CompoundTag getOversizedData(int x, int z) throws IOException { + public synchronized CompoundTag getOversizedData(int x, int z) throws IOException { // LinearPaper - make public Path file = getOversizedFile(x, z); try (DataInputStream out = new DataInputStream(new java.io.BufferedInputStream(new InflaterInputStream(Files.newInputStream(file))))) { return NbtIo.read((java.io.DataInput) out); @@ -1021,7 +1021,7 @@ public class RegionFile implements AutoCloseable, ca.spottedleaf.moonrise.patche } @Override - public final void moonrise$write(final RegionFile regionFile) throws IOException { + public final void moonrise$write(final org.stupidcraft.linearpaper.region.IRegionFile regionFile) throws IOException { // LinearPaper regionFile.write(this.pos, ByteBuffer.wrap(this.buf, 0, this.count)); } // Paper end - rewrite chunk system diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java index 79aca7e7cd2f860464657e77e935391642981fad..6721adad3d227acba8daba80b0b7e1df3bc0b046 100644 --- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java +++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java @@ -23,7 +23,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise public static final String ANVIL_EXTENSION = ".mca"; private static final int MAX_CACHE_SIZE = 256; - public final Long2ObjectLinkedOpenHashMap regionCache = new Long2ObjectLinkedOpenHashMap(); + public final Long2ObjectLinkedOpenHashMap regionCache = new Long2ObjectLinkedOpenHashMap(); // LinearPaper private final RegionStorageInfo info; private final Path folder; private final boolean sync; @@ -32,7 +32,11 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise private static final int REGION_SHIFT = 5; private static final int MAX_NON_EXISTING_CACHE = 1024 * 4; private final it.unimi.dsi.fastutil.longs.LongLinkedOpenHashSet nonExistingRegionFiles = new it.unimi.dsi.fastutil.longs.LongLinkedOpenHashSet(); + // Leaf start - Linear region format private static String getRegionFileName(final int chunkX, final int chunkZ) { + if (org.dreeam.leaf.config.modules.misc.RegionFormatConfig.regionFormatType == org.stupidcraft.linearpaper.region.EnumRegionFileExtension.LINEAR) { + return "r." + (chunkX >> REGION_SHIFT) + "." + (chunkZ >> REGION_SHIFT) + ".linear"; + } return "r." + (chunkX >> REGION_SHIFT) + "." + (chunkZ >> REGION_SHIFT) + ".mca"; } @@ -68,15 +72,15 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise } @Override - public synchronized final RegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ) { + public synchronized final org.stupidcraft.linearpaper.region.IRegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ) { // LinearPaper return this.regionCache.getAndMoveToFirst(ChunkPos.asLong(chunkX >> REGION_SHIFT, chunkZ >> REGION_SHIFT)); } @Override - public synchronized final RegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException { + public synchronized final org.stupidcraft.linearpaper.region.IRegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException { // LinearPaper final long key = ChunkPos.asLong(chunkX >> REGION_SHIFT, chunkZ >> REGION_SHIFT); - RegionFile ret = this.regionCache.getAndMoveToFirst(key); + org.stupidcraft.linearpaper.region.IRegionFile ret = this.regionCache.getAndMoveToFirst(key); // LinearPaper if (ret != null) { return ret; } @@ -100,7 +104,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise FileUtil.createDirectoriesSafe(this.folder); - ret = new RegionFile(this.info, regionPath, this.folder, this.sync); + ret = org.stupidcraft.linearpaper.region.IRegionFileFactory.getAbstractRegionFile(this.info, regionPath, this.folder, this.sync); // LinearPaper this.regionCache.putAndMoveToFirst(key, ret); @@ -119,7 +123,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise } final ChunkPos pos = new ChunkPos(chunkX, chunkZ); - final RegionFile regionFile = this.getRegionFile(pos); + final org.stupidcraft.linearpaper.region.IRegionFile regionFile = this.getRegionFile(pos); // LinearPaper // note: not required to keep regionfile loaded after this call, as the write param takes a regionfile as input // (and, the regionfile parameter is unused for writing until the write call) @@ -153,7 +157,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise ) throws IOException { final ChunkPos pos = new ChunkPos(chunkX, chunkZ); if (writeData.result() == ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO.RegionDataController.WriteData.WriteResult.DELETE) { - final RegionFile regionFile = this.moonrise$getRegionFileIfExists(chunkX, chunkZ); + final org.stupidcraft.linearpaper.region.IRegionFile regionFile = this.moonrise$getRegionFileIfExists(chunkX, chunkZ); // LinearPaper if (regionFile != null) { regionFile.clear(pos); } // else: didn't exist @@ -168,7 +172,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise public final ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO.RegionDataController.ReadData moonrise$readData( final int chunkX, final int chunkZ ) throws IOException { - final RegionFile regionFile = this.moonrise$getRegionFileIfExists(chunkX, chunkZ); + final org.stupidcraft.linearpaper.region.IRegionFile regionFile = this.moonrise$getRegionFileIfExists(chunkX, chunkZ); // LinearPaper final DataInputStream input = regionFile == null ? null : regionFile.getChunkDataInputStream(new ChunkPos(chunkX, chunkZ)); @@ -221,7 +225,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise @Nullable public static ChunkPos getRegionFileCoordinates(Path file) { String fileName = file.getFileName().toString(); - if (!fileName.startsWith("r.") || !fileName.endsWith(".mca")) { + if (!fileName.startsWith("r.") || !fileName.endsWith(".mca") || !fileName.endsWith(".linear")) { // LinearPaper return null; } @@ -250,12 +254,12 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise } // Paper start - rewrite chunk system - public RegionFile getRegionFile(ChunkPos chunkcoordintpair) throws IOException { + public org.stupidcraft.linearpaper.region.IRegionFile getRegionFile(ChunkPos chunkcoordintpair) throws IOException { // LinearPaper return this.getRegionFile(chunkcoordintpair, false); } // Paper end - rewrite chunk system - public RegionFile getRegionFile(ChunkPos chunkcoordintpair, boolean existingOnly) throws IOException { // CraftBukkit // Paper - public + public org.stupidcraft.linearpaper.region.IRegionFile getRegionFile(ChunkPos chunkcoordintpair, boolean existingOnly) throws IOException { // CraftBukkit // Paper - public // LinearPaper // Paper start - rewrite chunk system if (existingOnly) { return this.moonrise$getRegionFileIfExists(chunkcoordintpair.x, chunkcoordintpair.z); @@ -263,7 +267,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise synchronized (this) { final long key = ChunkPos.asLong(chunkcoordintpair.x >> REGION_SHIFT, chunkcoordintpair.z >> REGION_SHIFT); - RegionFile ret = this.regionCache.getAndMoveToFirst(key); + org.stupidcraft.linearpaper.region.IRegionFile ret = this.regionCache.getAndMoveToFirst(key); // LinearPaper if (ret != null) { return ret; } @@ -272,13 +276,13 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise this.regionCache.removeLast().close(); } - final Path regionPath = this.folder.resolve(getRegionFileName(chunkcoordintpair.x, chunkcoordintpair.z)); + final Path regionPath = this.folder.resolve(getRegionFileName(chunkcoordintpair.x, chunkcoordintpair.z)); // LinearPaper this.createRegionFile(key); FileUtil.createDirectoriesSafe(this.folder); - ret = new RegionFile(this.info, regionPath, this.folder, this.sync); + ret = org.stupidcraft.linearpaper.region.IRegionFileFactory.getAbstractRegionFile(this.info, regionPath, this.folder, this.sync); // LinearPaper this.regionCache.putAndMoveToFirst(key, ret); @@ -298,7 +302,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise // Gale end - branding changes } - private static CompoundTag readOversizedChunk(RegionFile regionfile, ChunkPos chunkCoordinate) throws IOException { + private static CompoundTag readOversizedChunk(org.stupidcraft.linearpaper.region.IRegionFile regionfile, ChunkPos chunkCoordinate) throws IOException { // LinearPaper synchronized (regionfile) { try (DataInputStream datainputstream = regionfile.getChunkDataInputStream(chunkCoordinate)) { CompoundTag oversizedData = regionfile.getOversizedData(chunkCoordinate.x, chunkCoordinate.z); @@ -333,7 +337,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise @Nullable public CompoundTag read(ChunkPos pos) throws IOException { // CraftBukkit start - SPIGOT-5680: There's no good reason to preemptively create files on read, save that for writing - RegionFile regionfile = this.getRegionFile(pos, true); + org.stupidcraft.linearpaper.region.IRegionFile regionfile = this.getRegionFile(pos, true); // LinearPaper if (regionfile == null) { return null; } @@ -397,7 +401,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise public void scanChunk(ChunkPos chunkPos, StreamTagVisitor scanner) throws IOException { // CraftBukkit start - SPIGOT-5680: There's no good reason to preemptively create files on read, save that for writing - RegionFile regionfile = this.getRegionFile(chunkPos, true); + org.stupidcraft.linearpaper.region.IRegionFile regionfile = this.getRegionFile(chunkPos, true); // LinearPaper if (regionfile == null) { return; } @@ -427,7 +431,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise } public void write(ChunkPos pos, @Nullable CompoundTag nbt) throws IOException { // Paper - rewrite chunk system - public - RegionFile regionfile = this.getRegionFile(pos, nbt == null); // CraftBukkit // Paper - rewrite chunk system + org.stupidcraft.linearpaper.region.IRegionFile regionfile = this.getRegionFile(pos, nbt == null); // CraftBukkit // Paper - rewrite chunk system // LinearPaper // Paper start - rewrite chunk system if (regionfile == null) { // if the RegionFile doesn't exist, no point in deleting from it @@ -471,7 +475,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise // Paper start - rewrite chunk system synchronized (this) { final ExceptionCollector exceptionCollector = new ExceptionCollector<>(); - for (final RegionFile regionFile : this.regionCache.values()) { + for (final org.stupidcraft.linearpaper.region.IRegionFile regionFile : this.regionCache.values()) { // LinearPaper try { regionFile.close(); } catch (final IOException ex) { @@ -488,7 +492,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise // Paper start - rewrite chunk system synchronized (this) { final ExceptionCollector exceptionCollector = new ExceptionCollector<>(); - for (final RegionFile regionFile : this.regionCache.values()) { + for (final org.stupidcraft.linearpaper.region.IRegionFile regionFile : this.regionCache.values()) { // LinearPaper try { regionFile.flush(); } catch (final IOException ex) { diff --git a/src/main/java/org/dreeam/leaf/config/modules/misc/RegionFormatConfig.java b/src/main/java/org/dreeam/leaf/config/modules/misc/RegionFormatConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..d94f8dd4a8682125ea356d36632e2b68e73d8af3 --- /dev/null +++ b/src/main/java/org/dreeam/leaf/config/modules/misc/RegionFormatConfig.java @@ -0,0 +1,62 @@ +package org.dreeam.leaf.config.modules.misc; + +import com.mojang.logging.LogUtils; +import org.dreeam.leaf.config.ConfigModules; +import org.dreeam.leaf.config.EnumConfigCategory; +import org.dreeam.leaf.config.annotations.DoNotLoad; +import org.slf4j.Logger; +import org.stupidcraft.linearpaper.region.EnumRegionFileExtension; + +public class RegionFormatConfig extends ConfigModules { + + public String getBasePath() { + return EnumConfigCategory.MISC.getBaseKeyName() + ".region-format-settings"; + } + + @DoNotLoad + private static final Logger logger = LogUtils.getLogger(); + @DoNotLoad + public static EnumRegionFileExtension regionFormatType; + + public static String regionFormatTypeName = "MCA"; + public static int linearCompressionLevel = 1; + public static boolean throwOnUnknownExtension = false; + public static int linearFlushFrequency = 5; + + @Override + public void onLoaded() { + config.addCommentRegionBased(getBasePath(), """ + Linear is a region file format that uses ZSTD compression instead of + ZLIB. + This format saves about 50% of disk space. + Read Documentation before using: https://github.com/xymb-endcrystalme/LinearRegionFileFormatTools + Disclaimer: This is an experimental feature, there is potential risk to lose chunk data. + So backup your server before switching to Linear.""", + """ + Linear 是一种使用 ZSTD 压缩而非 ZLIB 的区域文件格式. + 该格式可节省约 50% 的磁盘空间. + 使用前请阅读文档: https://github.com/xymb-endcrystalme/LinearRegionFileFormatTools + 免责声明: 实验性功能,有可能导致区块数据丢失. + 切换到Linear前请备份服务器."""); + + regionFormatTypeName = config.getString(getBasePath() + ".region-format", regionFormatTypeName, + config.pickStringRegionBased( + "Available region formats: MCA, LINEAR", + "可用格式: MCA, LINEAR")); + linearCompressionLevel = config.getInt(getBasePath() + ".linear-compress-level", linearCompressionLevel); + throwOnUnknownExtension = config().getBoolean(getBasePath() + ".throw-on-unknown-extension-detected", throwOnUnknownExtension); + linearFlushFrequency = config.getInt(getBasePath() + ".flush-interval-seconds", linearFlushFrequency); + + regionFormatType = EnumRegionFileExtension.fromName(regionFormatTypeName); + if (regionFormatType == EnumRegionFileExtension.UNKNOWN) { + logger.error("Unknown region file type {} ! Falling back to MCA format.", regionFormatTypeName); + regionFormatType = EnumRegionFileExtension.MCA; + } + + if (linearCompressionLevel > 23 || linearCompressionLevel < 1) { + logger.error("Linear region compression level should be between 1 and 22 in config: {}", linearCompressionLevel); + logger.error("Falling back to compression level 1."); + linearCompressionLevel = 1; + } + } +} diff --git a/src/main/java/org/stupidcraft/linearpaper/region/EnumRegionFileExtension.java b/src/main/java/org/stupidcraft/linearpaper/region/EnumRegionFileExtension.java new file mode 100644 index 0000000000000000000000000000000000000000..8eba172be9bce7cccd27d27bee2d6c6f917a9e4e --- /dev/null +++ b/src/main/java/org/stupidcraft/linearpaper/region/EnumRegionFileExtension.java @@ -0,0 +1,56 @@ +package org.stupidcraft.linearpaper.region; + +import org.jetbrains.annotations.Contract; +import org.jetbrains.annotations.NotNull; + +import java.util.Locale; + +public enum EnumRegionFileExtension { + LINEAR(".linear"), + MCA(".mca"), + UNKNOWN(null); + + private final String extensionName; + + EnumRegionFileExtension(String extensionName) { + this.extensionName = extensionName; + } + + public String getExtensionName() { + return this.extensionName; + } + + @Contract(pure = true) + public static EnumRegionFileExtension fromName(@NotNull String name) { + switch (name.toUpperCase(Locale.ROOT)) { + default -> { + return UNKNOWN; + } + + case "MCA" -> { + return MCA; + } + + case "LINEAR" -> { + return LINEAR; + } + } + } + + @Contract(pure = true) + public static EnumRegionFileExtension fromExtension(@NotNull String name) { + switch (name.toLowerCase()) { + case "mca" -> { + return MCA; + } + + case "linear" -> { + return LINEAR; + } + + default -> { + return UNKNOWN; + } + } + } +} diff --git a/src/main/java/org/stupidcraft/linearpaper/region/IRegionFile.java b/src/main/java/org/stupidcraft/linearpaper/region/IRegionFile.java new file mode 100644 index 0000000000000000000000000000000000000000..77a3629aeae5dfde37b8ea3da97ed008a30e1b94 --- /dev/null +++ b/src/main/java/org/stupidcraft/linearpaper/region/IRegionFile.java @@ -0,0 +1,28 @@ +package org.stupidcraft.linearpaper.region; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Path; + +import net.minecraft.nbt.CompoundTag; +import net.minecraft.world.level.ChunkPos; + +public interface IRegionFile { + Path getPath(); + void flush() throws IOException; + void clear(ChunkPos pos) throws IOException; + void close() throws IOException; + void setOversized(int x, int z, boolean b) throws IOException; + void write(ChunkPos pos, ByteBuffer buffer) throws IOException; + + boolean hasChunk(ChunkPos pos); + boolean doesChunkExist(ChunkPos pos) throws Exception; + boolean isOversized(int x, int z); + boolean recalculateHeader() throws IOException; + + DataOutputStream getChunkDataOutputStream(ChunkPos pos) throws IOException; + DataInputStream getChunkDataInputStream(ChunkPos pos) throws IOException; + CompoundTag getOversizedData(int x, int z) throws IOException; +} diff --git a/src/main/java/org/stupidcraft/linearpaper/region/IRegionFileFactory.java b/src/main/java/org/stupidcraft/linearpaper/region/IRegionFileFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..70552d63a84a4d3a73348d0dffacd89ca8f5df0f --- /dev/null +++ b/src/main/java/org/stupidcraft/linearpaper/region/IRegionFileFactory.java @@ -0,0 +1,52 @@ +package org.stupidcraft.linearpaper.region; + +import java.io.IOException; +import java.nio.file.Path; + +import net.minecraft.world.level.chunk.storage.RegionFile; +import net.minecraft.world.level.chunk.storage.RegionFileVersion; +import net.minecraft.world.level.chunk.storage.RegionStorageInfo; +import org.dreeam.leaf.config.modules.misc.RegionFormatConfig; +import org.jetbrains.annotations.Contract; +import org.jetbrains.annotations.NotNull; + +public class IRegionFileFactory { + @Contract("_, _, _, _ -> new") + public static @NotNull IRegionFile getAbstractRegionFile(RegionStorageInfo storageKey, Path directory, Path path, boolean dsync) throws IOException { + return getAbstractRegionFile(storageKey, directory, path, RegionFileVersion.getCompressionFormat(), dsync); + } + + @Contract("_, _, _, _, _ -> new") + public static @NotNull IRegionFile getAbstractRegionFile(RegionStorageInfo storageKey, Path directory, Path path, boolean dsync, boolean canRecalcHeader) throws IOException { + return getAbstractRegionFile(storageKey, directory, path, RegionFileVersion.getCompressionFormat(), dsync, canRecalcHeader); + } + + @Contract("_, _, _, _, _ -> new") + public static @NotNull IRegionFile getAbstractRegionFile(RegionStorageInfo storageKey, Path path, Path directory, RegionFileVersion compressionFormat, boolean dsync) throws IOException { + return getAbstractRegionFile(storageKey, path, directory, compressionFormat, dsync, true); + } + + @Contract("_, _, _, _, _, _ -> new") + public static @NotNull IRegionFile getAbstractRegionFile(RegionStorageInfo storageKey, @NotNull Path path, Path directory, RegionFileVersion compressionFormat, boolean dsync, boolean canRecalcHeader) throws IOException { + final String fullFileName = path.getFileName().toString(); + final String[] fullNameSplit = fullFileName.split("\\."); + final String extensionName = fullNameSplit[fullNameSplit.length - 1]; + switch (EnumRegionFileExtension.fromExtension(extensionName)) { + case UNKNOWN -> { + if (RegionFormatConfig.throwOnUnknownExtension) { + throw new IllegalArgumentException("Unknown region file extension for file: " + fullFileName + "!"); + } + + return new RegionFile(storageKey, path, directory, compressionFormat, dsync); + } + + case LINEAR -> { + return new LinearRegionFile(path, RegionFormatConfig.linearCompressionLevel); + } + + default -> { + return new RegionFile(storageKey, path, directory, compressionFormat, dsync); + } + } + } +} diff --git a/src/main/java/org/stupidcraft/linearpaper/region/LinearRegionFile.java b/src/main/java/org/stupidcraft/linearpaper/region/LinearRegionFile.java new file mode 100644 index 0000000000000000000000000000000000000000..d2cbdd75f02d377a7680288bb1154d51dbc6bc51 --- /dev/null +++ b/src/main/java/org/stupidcraft/linearpaper/region/LinearRegionFile.java @@ -0,0 +1,299 @@ +package org.stupidcraft.linearpaper.region; + +import com.github.luben.zstd.ZstdInputStream; +import com.github.luben.zstd.ZstdOutputStream; +import com.mojang.logging.LogUtils; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; + +import net.jpountz.lz4.LZ4Compressor; +import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4FastDecompressor; +import net.minecraft.nbt.CompoundTag; +import net.minecraft.world.level.ChunkPos; +import org.dreeam.leaf.config.modules.misc.RegionFormatConfig; +import org.slf4j.Logger; + +public class LinearRegionFile implements IRegionFile, AutoCloseable { + private static final long SUPERBLOCK = -4323716122432332390L; + private static final byte VERSION = 2; + private static final int HEADER_SIZE = 32; + private static final int FOOTER_SIZE = 8; + private static final Logger LOGGER = LogUtils.getLogger(); + private static final List SUPPORTED_VERSIONS = Arrays.asList((byte) 1, (byte) 2); + private final byte[][] buffer = new byte[1024][]; + private final int[] bufferUncompressedSize = new int[1024]; + private final int[] chunkTimestamps = new int[1024]; + private final LZ4Compressor compressor; + private final LZ4FastDecompressor decompressor; + private final int compressionLevel; + public boolean closed = false; + public Path path; + private volatile long lastFlushed = System.nanoTime(); + + public LinearRegionFile(Path file, int compression) throws IOException { + this.path = file; + this.compressionLevel = compression; + this.compressor = LZ4Factory.fastestInstance().fastCompressor(); + this.decompressor = LZ4Factory.fastestInstance().fastDecompressor(); + + File regionFile = new File(this.path.toString()); + + Arrays.fill(this.bufferUncompressedSize, 0); + + if (!regionFile.canRead()) return; + + try (FileInputStream fileStream = new FileInputStream(regionFile); + DataInputStream rawDataStream = new DataInputStream(fileStream)) { + + long superBlock = rawDataStream.readLong(); + if (superBlock != SUPERBLOCK) + throw new RuntimeException("Invalid superblock: " + superBlock + " in " + file); + + byte version = rawDataStream.readByte(); + if (!SUPPORTED_VERSIONS.contains(version)) + throw new RuntimeException("Invalid version: " + version + " in " + file); + + // Skip newestTimestamp (Long) + Compression level (Byte) + Chunk count (Short): Unused. + rawDataStream.skipBytes(11); + + int dataCount = rawDataStream.readInt(); + long fileLength = file.toFile().length(); + if (fileLength != HEADER_SIZE + dataCount + FOOTER_SIZE) + throw new IOException("Invalid file length: " + this.path + " " + fileLength + " " + (HEADER_SIZE + dataCount + FOOTER_SIZE)); + + rawDataStream.skipBytes(8); // Skip data hash (Long): Unused. + + byte[] rawCompressed = new byte[dataCount]; + rawDataStream.readFully(rawCompressed, 0, dataCount); + + superBlock = rawDataStream.readLong(); + if (superBlock != SUPERBLOCK) + throw new IOException("Footer superblock invalid " + this.path); + + try (DataInputStream dataStream = new DataInputStream(new ZstdInputStream(new ByteArrayInputStream(rawCompressed)))) { + + int[] starts = new int[1024]; + for (int i = 0; i < 1024; i++) { + starts[i] = dataStream.readInt(); + dataStream.skipBytes(4); // Skip timestamps (Int): Unused. + } + + for (int i = 0; i < 1024; i++) { + if (starts[i] > 0) { + int size = starts[i]; + byte[] b = new byte[size]; + dataStream.readFully(b, 0, size); + + int maxCompressedLength = this.compressor.maxCompressedLength(size); + byte[] compressed = new byte[maxCompressedLength]; + int compressedLength = this.compressor.compress(b, 0, size, compressed, 0, maxCompressedLength); + b = new byte[compressedLength]; + System.arraycopy(compressed, 0, b, 0, compressedLength); + + this.buffer[i] = b; + this.bufferUncompressedSize[i] = size; + } + } + } + } + } + + private static int getChunkIndex(int x, int z) { + return (x & 31) + ((z & 31) << 5); + } + + private static int getTimestamp() { + return (int) (System.currentTimeMillis() / 1000L); + } + + public void flush() throws IOException { + flushWrapper(); // sync + } + + public void flushWrapper() { + try { + save(); + } catch (IOException e) { + LOGGER.error("Failed to flush region file {}", path.toAbsolutePath(), e); + } + } + + public boolean doesChunkExist(ChunkPos pos) throws Exception { + throw new Exception("doesChunkExist is a stub"); + } + + private synchronized void save() throws IOException { + long timestamp = getTimestamp(); + short chunkCount = 0; + + File tempFile = new File(path.toString() + ".tmp"); + + try (FileOutputStream fileStream = new FileOutputStream(tempFile); + ByteArrayOutputStream zstdByteArray = new ByteArrayOutputStream(); + ZstdOutputStream zstdStream = new ZstdOutputStream(zstdByteArray, this.compressionLevel); + DataOutputStream zstdDataStream = new DataOutputStream(zstdStream); + DataOutputStream dataStream = new DataOutputStream(fileStream)) { + + dataStream.writeLong(SUPERBLOCK); + dataStream.writeByte(VERSION); + dataStream.writeLong(timestamp); + dataStream.writeByte(this.compressionLevel); + + ArrayList byteBuffers = new ArrayList<>(); + for (int i = 0; i < 1024; i++) { + if (this.bufferUncompressedSize[i] != 0) { + chunkCount += 1; + byte[] content = new byte[bufferUncompressedSize[i]]; + this.decompressor.decompress(buffer[i], 0, content, 0, bufferUncompressedSize[i]); + + byteBuffers.add(content); + } else byteBuffers.add(null); + } + for (int i = 0; i < 1024; i++) { + zstdDataStream.writeInt(this.bufferUncompressedSize[i]); // Write uncompressed size + zstdDataStream.writeInt(this.chunkTimestamps[i]); // Write timestamp + } + for (int i = 0; i < 1024; i++) { + if (byteBuffers.get(i) != null) + zstdDataStream.write(byteBuffers.get(i), 0, byteBuffers.get(i).length); + } + zstdDataStream.close(); + + dataStream.writeShort(chunkCount); + + byte[] compressed = zstdByteArray.toByteArray(); + + dataStream.writeInt(compressed.length); + dataStream.writeLong(0); + + dataStream.write(compressed, 0, compressed.length); + dataStream.writeLong(SUPERBLOCK); + + dataStream.flush(); + fileStream.getFD().sync(); + fileStream.getChannel().force(true); // Ensure atomicity on Btrfs + } + Files.move(tempFile.toPath(), this.path, StandardCopyOption.REPLACE_EXISTING); + this.lastFlushed = System.nanoTime(); + } + + public synchronized void write(ChunkPos pos, ByteBuffer buffer) { + try { + byte[] b = toByteArray(new ByteArrayInputStream(buffer.array())); + int uncompressedSize = b.length; + + int maxCompressedLength = this.compressor.maxCompressedLength(b.length); + byte[] compressed = new byte[maxCompressedLength]; + int compressedLength = this.compressor.compress(b, 0, b.length, compressed, 0, maxCompressedLength); + b = new byte[compressedLength]; + System.arraycopy(compressed, 0, b, 0, compressedLength); + + int index = getChunkIndex(pos.x, pos.z); + this.buffer[index] = b; + this.chunkTimestamps[index] = getTimestamp(); + this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] = uncompressedSize; + } catch (IOException e) { + LOGGER.error("Chunk write IOException {} {}", e, this.path); + } + + if ((System.nanoTime() - this.lastFlushed) >= TimeUnit.NANOSECONDS.toSeconds(RegionFormatConfig.linearFlushFrequency)) { + this.flushWrapper(); + } + } + + public DataOutputStream getChunkDataOutputStream(ChunkPos pos) { + return new DataOutputStream(new BufferedOutputStream(new ChunkBuffer(pos))); + } + + private byte[] toByteArray(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] tempBuffer = new byte[4096]; + + int length; + while ((length = in.read(tempBuffer)) >= 0) { + out.write(tempBuffer, 0, length); + } + + return out.toByteArray(); + } + + @Nullable + public synchronized DataInputStream getChunkDataInputStream(ChunkPos pos) { + if (this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] != 0) { + byte[] content = new byte[bufferUncompressedSize[getChunkIndex(pos.x, pos.z)]]; + this.decompressor.decompress(this.buffer[getChunkIndex(pos.x, pos.z)], 0, content, 0, bufferUncompressedSize[getChunkIndex(pos.x, pos.z)]); + return new DataInputStream(new ByteArrayInputStream(content)); + } + return null; + } + + public void clear(ChunkPos pos) { + int i = getChunkIndex(pos.x, pos.z); + this.buffer[i] = null; + this.bufferUncompressedSize[i] = 0; + this.chunkTimestamps[i] = getTimestamp(); + this.flushWrapper(); + } + + public Path getPath() { + return this.path; + } + + public boolean hasChunk(ChunkPos pos) { + return this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] > 0; + } + + public void close() throws IOException { + if (closed) return; + closed = true; + flush(); // sync + } + + public boolean recalculateHeader() { + return false; + } + + public void setOversized(int x, int z, boolean something) { + } + + public CompoundTag getOversizedData(int x, int z) throws IOException { + throw new IOException("getOversizedData is a stub " + this.path); + } + + public boolean isOversized(int x, int z) { + return false; + } + + private class ChunkBuffer extends ByteArrayOutputStream { + private final ChunkPos pos; + + public ChunkBuffer(ChunkPos chunkcoordintpair) { + super(); + this.pos = chunkcoordintpair; + } + + public void close() { + ByteBuffer bytebuffer = ByteBuffer.wrap(this.buf, 0, this.count); + LinearRegionFile.this.write(this.pos, bytebuffer); + } + } +}