diff --git a/README.md b/README.md index 85e6ac08..86201e24 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ - **Configurable UseItem distance** for anarchy servers - **Mod Protocols** support - **More customized** relying on features of [Purpur](https://github.com/PurpurMC/Purpur) + - Support for **Linear region file format** - **Maintenance friendly**, integrating with [Sentry](https://sentry.io/welcome/) of [Pufferfish](https://github.com/pufferfish-gg/Pufferfish) to easy track all errors coming from your server in excruciating detail - And more... diff --git a/patches/server/0080-Linear-region-file-format.patch b/patches/server/0080-Linear-region-file-format.patch new file mode 100644 index 00000000..72773597 --- /dev/null +++ b/patches/server/0080-Linear-region-file-format.patch @@ -0,0 +1,824 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: HaHaWTH <102713261+HaHaWTH@users.noreply.github.com> +Date: Sun, 30 Jun 2024 00:35:19 +0800 +Subject: [PATCH] Linear-region-file-format + + +diff --git a/build.gradle.kts b/build.gradle.kts +index 49a3b6892b3e075f393e4ed8e4e0d5afc4798db5..d83e9365287a497a997cdb0e5d608635a739528c 100644 +--- a/build.gradle.kts ++++ b/build.gradle.kts +@@ -28,6 +28,11 @@ dependencies { + implementation("com.github.ben-manes.caffeine:caffeine:3.1.8") + // Leaf end + ++ // LinearPaper start ++ implementation("com.github.luben:zstd-jni:1.5.6-3") ++ implementation("org.lz4:lz4-java:1.8.0") ++ // LinearPaper end ++ + // Paper start + implementation("org.jline:jline-terminal-jansi:3.26.1") // Leaf - Bump Dependencies + implementation("net.minecrell:terminalconsoleappender:1.3.0") +diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java +index 73df26b27146bbad2106d57b22dd3c792ed3dd1d..d2f209dd61f4412478a4b8fbe3489787d30bf74b 100644 +--- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java ++++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java +@@ -7,8 +7,8 @@ public interface ChunkSystemRegionFileStorage { + + public boolean moonrise$doesRegionFileNotExistNoIO(final int chunkX, final int chunkZ); + +- public RegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ); ++ public org.stupidcraft.linearpaper.region.IRegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ); // LinearPaper // Leaf + +- public RegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException; ++ public org.stupidcraft.linearpaper.region.IRegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException; // LinearPaper // Leaf + + } +diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/RegionFileIOThread.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/RegionFileIOThread.java +index c833f78d083b8f661087471c35bc90f65af1b525..25bd2a98f494f9b3d16032ccee70f2e36ecf1069 100644 +--- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/RegionFileIOThread.java ++++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/RegionFileIOThread.java +@@ -1042,9 +1042,9 @@ public final class RegionFileIOThread extends PrioritisedQueueExecutorThread { + return ((ChunkSystemRegionFileStorage)(Object)this.getCache()).moonrise$doesRegionFileNotExistNoIO(chunkX, chunkZ); + } + +- public T computeForRegionFile(final int chunkX, final int chunkZ, final boolean existingOnly, final Function function) { ++ public T computeForRegionFile(final int chunkX, final int chunkZ, final boolean existingOnly, final Function function) { // LinearPaper + final RegionFileStorage cache = this.getCache(); +- final RegionFile regionFile; ++ final org.stupidcraft.linearpaper.region.IRegionFile regionFile; // LinearPaper + synchronized (cache) { + try { + if (existingOnly) { +@@ -1060,9 +1060,9 @@ public final class RegionFileIOThread extends PrioritisedQueueExecutorThread { + } + } + +- public T computeForRegionFileIfLoaded(final int chunkX, final int chunkZ, final Function function) { ++ public T computeForRegionFileIfLoaded(final int chunkX, final int chunkZ, final Function function) { // LinearPaper + final RegionFileStorage cache = this.getCache(); +- final RegionFile regionFile; ++ final org.stupidcraft.linearpaper.region.IRegionFile regionFile; // LinearPaper + + synchronized (cache) { + regionFile = ((ChunkSystemRegionFileStorage)(Object)cache).moonrise$getRegionFileIfLoaded(chunkX, chunkZ); +diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java +index ef14203da6f9d0f5cad4335c98095522209023cd..f9704361f1d480071ae07b0cb48c9851396f8ffa 100644 +--- a/src/main/java/net/minecraft/server/MinecraftServer.java ++++ b/src/main/java/net/minecraft/server/MinecraftServer.java +@@ -904,10 +904,10 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop> progressMap = Reference2FloatMaps.synchronize(new Reference2FloatOpenHashMap()); + volatile Component status = Component.translatable("optimizeWorld.stage.counting"); +- static final Pattern REGEX = Pattern.compile("^r\\.(-?[0-9]+)\\.(-?[0-9]+)\\.mca$"); ++ static final Pattern REGEX = Pattern.compile("^r\\.(-?[0-9]+)\\.(-?[0-9]+)\\.(linear | mca)$"); // LinearPaper + final DimensionDataStorage overworldDataStorage; + + public WorldUpgrader(LevelStorageSource.LevelStorageAccess session, DataFixer dataFixer, RegistryAccess dynamicRegistryManager, boolean eraseCache, boolean recreateRegionFiles) { +@@ -394,7 +394,7 @@ public class WorldUpgrader { + + private static List getAllChunkPositions(RegionStorageInfo key, Path regionDirectory) { + File[] afile = regionDirectory.toFile().listFiles((file, s) -> { +- return s.endsWith(".mca"); ++ return s.endsWith(".linear") || s.endsWith(".mca"); // LinearPaper + }); + + if (afile == null) { +@@ -414,7 +414,7 @@ public class WorldUpgrader { + List list1 = Lists.newArrayList(); + + try { +- RegionFile regionfile = new RegionFile(key, file.toPath(), regionDirectory, true); ++ org.stupidcraft.linearpaper.region.IRegionFile regionfile = org.stupidcraft.linearpaper.region.IRegionFileFactory.getAbstractRegionFile(key, file.toPath(), regionDirectory, true); // LinearPaper + + try { + for (int i1 = 0; i1 < 32; ++i1) { +@@ -477,7 +477,7 @@ public class WorldUpgrader { + + protected abstract boolean tryProcessOnePosition(T storage, ChunkPos chunkPos, ResourceKey worldKey); + +- private void onFileFinished(RegionFile regionFile) { ++ private void onFileFinished(org.stupidcraft.linearpaper.region.IRegionFile regionFile) { // LinearPaper + if (WorldUpgrader.this.recreateRegionFiles) { + if (this.previousWriteFuture != null) { + this.previousWriteFuture.join(); +@@ -502,7 +502,7 @@ public class WorldUpgrader { + } + } + +- static record FileToUpgrade(RegionFile file, List chunksToUpgrade) { ++ static record FileToUpgrade(org.stupidcraft.linearpaper.region.IRegionFile file, List chunksToUpgrade) { // LinearPaper + + } + +diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java +index e761b63eebc1e76b2bb1cb887d83d0b63ad6ec90..18c06a3a261d6f1488c64a9e77cc88e6eaabf263 100644 +--- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java ++++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java +@@ -28,7 +28,7 @@ import net.minecraft.nbt.NbtIo; // Paper + import net.minecraft.world.level.ChunkPos; + import org.slf4j.Logger; + +-public class RegionFile implements AutoCloseable { ++public class RegionFile implements org.stupidcraft.linearpaper.region.IRegionFile, AutoCloseable { // LinearPaper + + private static final Logger LOGGER = LogUtils.getLogger(); + private static final int SECTOR_BYTES = 4096; +@@ -465,10 +465,10 @@ public class RegionFile implements AutoCloseable { + private static int getChunkIndex(int x, int z) { + return (x & 31) + (z & 31) * 32; + } +- synchronized boolean isOversized(int x, int z) { ++ public synchronized boolean isOversized(int x, int z) { // LinearPaper - make public + return this.oversized[getChunkIndex(x, z)] == 1; + } +- synchronized void setOversized(int x, int z, boolean oversized) throws IOException { ++ public synchronized void setOversized(int x, int z, boolean oversized) throws IOException { // LinearPaper - make public + final int offset = getChunkIndex(x, z); + boolean previous = this.oversized[offset] == 1; + this.oversized[offset] = (byte) (oversized ? 1 : 0); +@@ -507,7 +507,7 @@ public class RegionFile implements AutoCloseable { + return this.path.getParent().resolve(this.path.getFileName().toString().replaceAll("\\.mca$", "") + "_oversized_" + x + "_" + z + ".nbt"); + } + +- synchronized CompoundTag getOversizedData(int x, int z) throws IOException { ++ public synchronized CompoundTag getOversizedData(int x, int z) throws IOException { // LinearPaper - make public + Path file = getOversizedFile(x, z); + try (DataInputStream out = new DataInputStream(new java.io.BufferedInputStream(new InflaterInputStream(Files.newInputStream(file))))) { + return NbtIo.read((java.io.DataInput) out); +diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java +index e8f0f0e2b3afd305afb6cf4d41f4fc31bf2bc68d..6c565c70de9a63ba897e582425e775fbe29dbb38 100644 +--- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java ++++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java +@@ -21,7 +21,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + + public static final String ANVIL_EXTENSION = ".mca"; + private static final int MAX_CACHE_SIZE = 256; +- public final Long2ObjectLinkedOpenHashMap regionCache = new Long2ObjectLinkedOpenHashMap(); ++ public final Long2ObjectLinkedOpenHashMap regionCache = new Long2ObjectLinkedOpenHashMap(); // LinearPaper + private final RegionStorageInfo info; + private final Path folder; + private final boolean sync; +@@ -30,7 +30,11 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + private static final int REGION_SHIFT = 5; + private static final int MAX_NON_EXISTING_CACHE = 1024 * 64; + private final it.unimi.dsi.fastutil.longs.LongLinkedOpenHashSet nonExistingRegionFiles = new it.unimi.dsi.fastutil.longs.LongLinkedOpenHashSet(MAX_NON_EXISTING_CACHE+1); ++ // Leaf start - Linear region format + private static String getRegionFileName(final int chunkX, final int chunkZ) { ++ if (org.dreeam.leaf.config.modules.misc.RegionFormatConfig.regionFormatType == org.stupidcraft.linearpaper.region.EnumRegionFileExtension.LINEAR) { ++ return "r." + (chunkX >> REGION_SHIFT) + "." + (chunkZ >> REGION_SHIFT) + ".linear"; ++ } + return "r." + (chunkX >> REGION_SHIFT) + "." + (chunkZ >> REGION_SHIFT) + ".mca"; + } + +@@ -66,15 +70,15 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + } + + @Override +- public synchronized final RegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ) { ++ public synchronized final org.stupidcraft.linearpaper.region.IRegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ) { // LinearPaper + return this.regionCache.getAndMoveToFirst(ChunkPos.asLong(chunkX >> REGION_SHIFT, chunkZ >> REGION_SHIFT)); + } + + @Override +- public synchronized final RegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException { ++ public synchronized final org.stupidcraft.linearpaper.region.IRegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException { // LinearPaper + final long key = ChunkPos.asLong(chunkX >> REGION_SHIFT, chunkZ >> REGION_SHIFT); + +- RegionFile ret = this.regionCache.getAndMoveToFirst(key); ++ org.stupidcraft.linearpaper.region.IRegionFile ret = this.regionCache.getAndMoveToFirst(key); // LinearPaper + if (ret != null) { + return ret; + } +@@ -98,7 +102,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + + FileUtil.createDirectoriesSafe(this.folder); + +- ret = new RegionFile(this.info, regionPath, this.folder, this.sync); ++ ret = org.stupidcraft.linearpaper.region.IRegionFileFactory.getAbstractRegionFile(this.info, regionPath, this.folder, this.sync); // LinearPaper + + this.regionCache.putAndMoveToFirst(key, ret); + +@@ -112,7 +116,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + this.info = storageKey; + } + +- public RegionFile getRegionFile(ChunkPos chunkcoordintpair, boolean existingOnly) throws IOException { // CraftBukkit // Paper - public ++ public org.stupidcraft.linearpaper.region.IRegionFile getRegionFile(ChunkPos chunkcoordintpair, boolean existingOnly) throws IOException { // CraftBukkit // Paper - public // LinearPaper + // Paper start - rewrite chunk system + if (existingOnly) { + return this.moonrise$getRegionFileIfExists(chunkcoordintpair.x, chunkcoordintpair.z); +@@ -120,7 +124,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + synchronized (this) { + final long key = ChunkPos.asLong(chunkcoordintpair.x >> REGION_SHIFT, chunkcoordintpair.z >> REGION_SHIFT); + +- RegionFile ret = this.regionCache.getAndMoveToFirst(key); ++ org.stupidcraft.linearpaper.region.IRegionFile ret = this.regionCache.getAndMoveToFirst(key); // LinearPaper + if (ret != null) { + return ret; + } +@@ -129,13 +133,13 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + this.regionCache.removeLast().close(); + } + +- final Path regionPath = this.folder.resolve(getRegionFileName(chunkcoordintpair.x, chunkcoordintpair.z)); ++ final Path regionPath = this.folder.resolve(getRegionFileName(chunkcoordintpair.x, chunkcoordintpair.z)); // LinearPaper + + this.createRegionFile(key); + + FileUtil.createDirectoriesSafe(this.folder); + +- ret = new RegionFile(this.info, regionPath, this.folder, this.sync); ++ ret = org.stupidcraft.linearpaper.region.IRegionFileFactory.getAbstractRegionFile(this.info, regionPath, this.folder, this.sync); // LinearPaper + + this.regionCache.putAndMoveToFirst(key, ret); + +@@ -155,7 +159,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + // Gale end - branding changes + } + +- private static CompoundTag readOversizedChunk(RegionFile regionfile, ChunkPos chunkCoordinate) throws IOException { ++ private static CompoundTag readOversizedChunk(org.stupidcraft.linearpaper.region.IRegionFile regionfile, ChunkPos chunkCoordinate) throws IOException { // LinearPaper + synchronized (regionfile) { + try (DataInputStream datainputstream = regionfile.getChunkDataInputStream(chunkCoordinate)) { + CompoundTag oversizedData = regionfile.getOversizedData(chunkCoordinate.x, chunkCoordinate.z); +@@ -190,7 +194,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + @Nullable + public CompoundTag read(ChunkPos pos) throws IOException { + // CraftBukkit start - SPIGOT-5680: There's no good reason to preemptively create files on read, save that for writing +- RegionFile regionfile = this.getRegionFile(pos, true); ++ org.stupidcraft.linearpaper.region.IRegionFile regionfile = this.getRegionFile(pos, true); // LinearPaper + if (regionfile == null) { + return null; + } +@@ -241,7 +245,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + + public void scanChunk(ChunkPos chunkPos, StreamTagVisitor scanner) throws IOException { + // CraftBukkit start - SPIGOT-5680: There's no good reason to preemptively create files on read, save that for writing +- RegionFile regionfile = this.getRegionFile(chunkPos, true); ++ org.stupidcraft.linearpaper.region.IRegionFile regionfile = this.getRegionFile(chunkPos, true); // LinearPaper + if (regionfile == null) { + return; + } +@@ -271,7 +275,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + } + + public void write(ChunkPos pos, @Nullable CompoundTag nbt) throws IOException { // Paper - public +- RegionFile regionfile = this.getRegionFile(pos, nbt == null); // CraftBukkit // Paper - rewrite chunk system ++ org.stupidcraft.linearpaper.region.IRegionFile regionfile = this.getRegionFile(pos, nbt == null); // CraftBukkit // Paper - rewrite chunk system // LinearPaper + // Paper start - rewrite chunk system + if (regionfile == null) { + // if the RegionFile doesn't exist, no point in deleting from it +@@ -330,7 +334,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + // Paper start - rewrite chunk system + synchronized (this) { + final ExceptionCollector exceptionCollector = new ExceptionCollector<>(); +- for (final RegionFile regionFile : this.regionCache.values()) { ++ for (final org.stupidcraft.linearpaper.region.IRegionFile regionFile : this.regionCache.values()) { // LinearPaper + try { + regionFile.close(); + } catch (final IOException ex) { +@@ -347,7 +351,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise + // Paper start - rewrite chunk system + synchronized (this) { + final ExceptionCollector exceptionCollector = new ExceptionCollector<>(); +- for (final RegionFile regionFile : this.regionCache.values()) { ++ for (final org.stupidcraft.linearpaper.region.IRegionFile regionFile : this.regionCache.values()) { // LinearPaper + try { + regionFile.flush(); + } catch (final IOException ex) { +diff --git a/src/main/java/org/dreeam/leaf/config/modules/misc/RegionFormatConfig.java b/src/main/java/org/dreeam/leaf/config/modules/misc/RegionFormatConfig.java +new file mode 100644 +index 0000000000000000000000000000000000000000..9efa36f9a18fafceb4fb184380d73e4271659a4f +--- /dev/null ++++ b/src/main/java/org/dreeam/leaf/config/modules/misc/RegionFormatConfig.java +@@ -0,0 +1,57 @@ ++package org.dreeam.leaf.config.modules.misc; ++ ++import com.electronwill.nightconfig.core.file.CommentedFileConfig; ++import com.mojang.logging.LogUtils; ++import org.dreeam.leaf.config.ConfigInfo; ++import org.dreeam.leaf.config.DoNotLoad; ++import org.dreeam.leaf.config.EnumConfigCategory; ++import org.dreeam.leaf.config.IConfigModule; ++import org.slf4j.Logger; ++import org.stupidcraft.linearpaper.region.EnumRegionFileExtension; ++ ++public class RegionFormatConfig implements IConfigModule { ++ @ConfigInfo(baseName = "region_format") ++ public static String regionFormatTypeName = "MCA"; ++ @DoNotLoad ++ public static EnumRegionFileExtension regionFormatType = EnumRegionFileExtension.LINEAR; ++ @ConfigInfo(baseName = "linear_compress_level") ++ public static int linearCompressionLevel = 1; ++ @ConfigInfo(baseName = "thrown_on_unknown_extension_detected") ++ public static boolean throwOnUnknownExtension = false; ++ @ConfigInfo(baseName = "flush_interval_seconds") ++ public static int linearFlushFrequency = 5; ++ ++ @DoNotLoad ++ private static final Logger logger = LogUtils.getLogger(); ++ ++ @Override ++ public EnumConfigCategory getCategory() { ++ return EnumConfigCategory.MISC; ++ } ++ ++ @Override ++ public String getBaseName() { ++ return "region_format_settings"; ++ } ++ ++ @Override ++ public void onLoaded(CommentedFileConfig config) { ++ config.setComment("misc.region_format_settings", """ ++ Linear is a region file format that uses ZSTD compression instead of ++ ZLIB. ++ This format saves about 50% of disk space. ++ Documentation: https://github.com/xymb-endcrystalme/LinearRegionFileFormatTools ++ """); ++ regionFormatType = EnumRegionFileExtension.fromName(regionFormatTypeName); ++ if (regionFormatType == EnumRegionFileExtension.UNKNOWN) { ++ logger.error("Unknown region file type {} ! Falling back to MCA file.", regionFormatTypeName); ++ regionFormatType = EnumRegionFileExtension.MCA; ++ } ++ ++ if (linearCompressionLevel > 23 || linearCompressionLevel < 1) { ++ logger.error("Linear region compression level should be between 1 and 22 in config: {}", linearCompressionLevel); ++ logger.error("Falling back to compression level 1."); ++ linearCompressionLevel = 1; ++ } ++ } ++} +diff --git a/src/main/java/org/stupidcraft/linearpaper/region/EnumRegionFileExtension.java b/src/main/java/org/stupidcraft/linearpaper/region/EnumRegionFileExtension.java +new file mode 100644 +index 0000000000000000000000000000000000000000..d4ce2d17a7ab258cac3b91065ab3f56fba20d1b1 +--- /dev/null ++++ b/src/main/java/org/stupidcraft/linearpaper/region/EnumRegionFileExtension.java +@@ -0,0 +1,56 @@ ++package org.stupidcraft.linearpaper.region; ++ ++import org.jetbrains.annotations.Contract; ++import org.jetbrains.annotations.NotNull; ++ ++import java.util.Locale; ++ ++public enum EnumRegionFileExtension { ++ LINEAR(".linear"), ++ MCA(".mca"), ++ UNKNOWN(null); ++ ++ private final String extensionName; ++ ++ EnumRegionFileExtension(String extensionName) { ++ this.extensionName = extensionName; ++ } ++ ++ public String getExtensionName() { ++ return this.extensionName; ++ } ++ ++ @Contract(pure = true) ++ public static EnumRegionFileExtension fromName(@NotNull String name) { ++ switch (name.toUpperCase(Locale.ROOT)) { ++ default -> { ++ return UNKNOWN; ++ } ++ ++ case "MCA" -> { ++ return MCA; ++ } ++ ++ case "LINEAR" -> { ++ return LINEAR; ++ } ++ } ++ } ++ ++ @Contract(pure = true) ++ public static EnumRegionFileExtension fromExtension(@NotNull String name) { ++ switch (name) { ++ case "mca" -> { ++ return MCA; ++ } ++ ++ case "linear" -> { ++ return LINEAR; ++ } ++ ++ default -> { ++ return UNKNOWN; ++ } ++ } ++ } ++} +\ No newline at end of file +diff --git a/src/main/java/org/stupidcraft/linearpaper/region/IRegionFile.java b/src/main/java/org/stupidcraft/linearpaper/region/IRegionFile.java +new file mode 100644 +index 0000000000000000000000000000000000000000..17db1bd619bc417cbd034e204600abbc506256ae +--- /dev/null ++++ b/src/main/java/org/stupidcraft/linearpaper/region/IRegionFile.java +@@ -0,0 +1,25 @@ ++package org.stupidcraft.linearpaper.region; ++ ++import java.io.DataInputStream; ++import java.io.DataOutputStream; ++import java.io.IOException; ++import java.nio.file.Path; ++ ++import net.minecraft.nbt.CompoundTag; ++import net.minecraft.world.level.ChunkPos; ++ ++public interface IRegionFile { ++ Path getPath(); ++ void flush() throws IOException; ++ void clear(ChunkPos pos) throws IOException; ++ void close() throws IOException; ++ void setOversized(int x, int z, boolean b) throws IOException; ++ ++ boolean hasChunk(ChunkPos pos); ++ boolean doesChunkExist(ChunkPos pos) throws Exception; ++ boolean isOversized(int x, int z); ++ ++ DataOutputStream getChunkDataOutputStream(ChunkPos pos) throws IOException; ++ DataInputStream getChunkDataInputStream(ChunkPos pos) throws IOException; ++ CompoundTag getOversizedData(int x, int z) throws IOException; ++} +diff --git a/src/main/java/org/stupidcraft/linearpaper/region/IRegionFileFactory.java b/src/main/java/org/stupidcraft/linearpaper/region/IRegionFileFactory.java +new file mode 100644 +index 0000000000000000000000000000000000000000..08c9b5d31408ddcb9dce4244cae8dfb22c84d20a +--- /dev/null ++++ b/src/main/java/org/stupidcraft/linearpaper/region/IRegionFileFactory.java +@@ -0,0 +1,52 @@ ++package org.stupidcraft.linearpaper.region; ++ ++import java.io.IOException; ++import java.nio.file.Path; ++ ++import net.minecraft.world.level.chunk.storage.RegionFile; ++import net.minecraft.world.level.chunk.storage.RegionFileVersion; ++import net.minecraft.world.level.chunk.storage.RegionStorageInfo; ++import org.dreeam.leaf.config.modules.misc.RegionFormatConfig; ++import org.jetbrains.annotations.Contract; ++import org.jetbrains.annotations.NotNull; ++ ++public class IRegionFileFactory { ++ @Contract("_, _, _, _ -> new") ++ public static @NotNull IRegionFile getAbstractRegionFile(RegionStorageInfo storageKey, Path directory, Path path, boolean dsync) throws IOException { ++ return getAbstractRegionFile(storageKey, directory, path, RegionFileVersion.getCompressionFormat(), dsync); ++ } ++ ++ @Contract("_, _, _, _, _ -> new") ++ public static @NotNull IRegionFile getAbstractRegionFile(RegionStorageInfo storageKey, Path directory, Path path, boolean dsync, boolean canRecalcHeader) throws IOException { ++ return getAbstractRegionFile(storageKey, directory, path, RegionFileVersion.getCompressionFormat(), dsync, canRecalcHeader); ++ } ++ ++ @Contract("_, _, _, _, _ -> new") ++ public static @NotNull IRegionFile getAbstractRegionFile(RegionStorageInfo storageKey, Path path, Path directory, RegionFileVersion compressionFormat, boolean dsync) throws IOException { ++ return getAbstractRegionFile(storageKey, path, directory, compressionFormat, dsync, true); ++ } ++ ++ @Contract("_, _, _, _, _, _ -> new") ++ public static @NotNull IRegionFile getAbstractRegionFile(RegionStorageInfo storageKey, @NotNull Path path, Path directory, RegionFileVersion compressionFormat, boolean dsync, boolean canRecalcHeader) throws IOException { ++ final String fullFileName = path.getFileName().toString(); ++ final String[] fullNameSplit = fullFileName.split("\\."); ++ final String extensionName = fullNameSplit[fullNameSplit.length - 1]; ++ switch (EnumRegionFileExtension.fromExtension(extensionName)) { ++ case UNKNOWN -> { ++ if (RegionFormatConfig.throwOnUnknownExtension) { ++ throw new IllegalArgumentException("Unknown region file extension for file: " + fullFileName +"!"); ++ } ++ ++ return new RegionFile(storageKey, path, directory, compressionFormat, dsync); ++ } ++ ++ case LINEAR -> { ++ return new LinearRegionFile(path, RegionFormatConfig.linearCompressionLevel); ++ } ++ ++ default -> { ++ return new RegionFile(storageKey, path, directory, compressionFormat, dsync); ++ } ++ } ++ } ++} +\ No newline at end of file +diff --git a/src/main/java/org/stupidcraft/linearpaper/region/LinearRegionFile.java b/src/main/java/org/stupidcraft/linearpaper/region/LinearRegionFile.java +new file mode 100644 +index 0000000000000000000000000000000000000000..c141fc6e4f4ba9bb9971b4f33886ed206b768e86 +--- /dev/null ++++ b/src/main/java/org/stupidcraft/linearpaper/region/LinearRegionFile.java +@@ -0,0 +1,294 @@ ++package org.stupidcraft.linearpaper.region; ++ ++import com.github.luben.zstd.ZstdInputStream; ++import com.github.luben.zstd.ZstdOutputStream; ++import com.mojang.logging.LogUtils; ++import java.io.BufferedOutputStream; ++import java.io.ByteArrayInputStream; ++import java.io.ByteArrayOutputStream; ++import java.io.DataInputStream; ++import java.io.DataOutputStream; ++import java.io.File; ++import java.io.FileInputStream; ++import java.io.FileOutputStream; ++import java.io.IOException; ++import java.io.InputStream; ++import java.nio.ByteBuffer; ++import java.nio.file.Files; ++import java.nio.file.Path; ++import java.nio.file.StandardCopyOption; ++import java.util.ArrayList; ++import java.util.Arrays; ++import java.util.List; ++import java.util.concurrent.TimeUnit; ++import javax.annotation.Nullable; ++ ++import net.jpountz.lz4.LZ4Compressor; ++import net.jpountz.lz4.LZ4Factory; ++import net.jpountz.lz4.LZ4FastDecompressor; ++import net.minecraft.nbt.CompoundTag; ++import net.minecraft.world.level.ChunkPos; ++import org.dreeam.leaf.config.modules.misc.RegionFormatConfig; ++import org.slf4j.Logger; ++ ++public class LinearRegionFile implements IRegionFile, AutoCloseable { ++ private static final long SUPERBLOCK = -4323716122432332390L; ++ private static final byte VERSION = 2; ++ private static final int HEADER_SIZE = 32; ++ private static final int FOOTER_SIZE = 8; ++ private static final Logger LOGGER = LogUtils.getLogger(); ++ private static final List SUPPORTED_VERSIONS = Arrays.asList((byte) 1, (byte) 2); ++ private final byte[][] buffer = new byte[1024][]; ++ private final int[] bufferUncompressedSize = new int[1024]; ++ private final int[] chunkTimestamps = new int[1024]; ++ private final LZ4Compressor compressor; ++ private final LZ4FastDecompressor decompressor; ++ private final int compressionLevel; ++ public boolean closed = false; ++ public Path path; ++ private volatile long lastFlushed = System.nanoTime(); ++ ++ public LinearRegionFile(Path file, int compression) throws IOException { ++ this.path = file; ++ this.compressionLevel = compression; ++ this.compressor = LZ4Factory.fastestInstance().fastCompressor(); ++ this.decompressor = LZ4Factory.fastestInstance().fastDecompressor(); ++ ++ File regionFile = new File(this.path.toString()); ++ ++ Arrays.fill(this.bufferUncompressedSize, 0); ++ ++ if (!regionFile.canRead()) return; ++ ++ try (FileInputStream fileStream = new FileInputStream(regionFile); ++ DataInputStream rawDataStream = new DataInputStream(fileStream)) { ++ ++ long superBlock = rawDataStream.readLong(); ++ if (superBlock != SUPERBLOCK) ++ throw new RuntimeException("Invalid superblock: " + superBlock + " in " + file); ++ ++ byte version = rawDataStream.readByte(); ++ if (!SUPPORTED_VERSIONS.contains(version)) ++ throw new RuntimeException("Invalid version: " + version + " in " + file); ++ ++ // Skip newestTimestamp (Long) + Compression level (Byte) + Chunk count (Short): Unused. ++ rawDataStream.skipBytes(11); ++ ++ int dataCount = rawDataStream.readInt(); ++ long fileLength = file.toFile().length(); ++ if (fileLength != HEADER_SIZE + dataCount + FOOTER_SIZE) ++ throw new IOException("Invalid file length: " + this.path + " " + fileLength + " " + (HEADER_SIZE + dataCount + FOOTER_SIZE)); ++ ++ rawDataStream.skipBytes(8); // Skip data hash (Long): Unused. ++ ++ byte[] rawCompressed = new byte[dataCount]; ++ rawDataStream.readFully(rawCompressed, 0, dataCount); ++ ++ superBlock = rawDataStream.readLong(); ++ if (superBlock != SUPERBLOCK) ++ throw new IOException("Footer superblock invalid " + this.path); ++ ++ try (DataInputStream dataStream = new DataInputStream(new ZstdInputStream(new ByteArrayInputStream(rawCompressed)))) { ++ ++ int[] starts = new int[1024]; ++ for (int i = 0; i < 1024; i++) { ++ starts[i] = dataStream.readInt(); ++ dataStream.skipBytes(4); // Skip timestamps (Int): Unused. ++ } ++ ++ for (int i = 0; i < 1024; i++) { ++ if (starts[i] > 0) { ++ int size = starts[i]; ++ byte[] b = new byte[size]; ++ dataStream.readFully(b, 0, size); ++ ++ int maxCompressedLength = this.compressor.maxCompressedLength(size); ++ byte[] compressed = new byte[maxCompressedLength]; ++ int compressedLength = this.compressor.compress(b, 0, size, compressed, 0, maxCompressedLength); ++ b = new byte[compressedLength]; ++ System.arraycopy(compressed, 0, b, 0, compressedLength); ++ ++ this.buffer[i] = b; ++ this.bufferUncompressedSize[i] = size; ++ } ++ } ++ } ++ } ++ } ++ ++ private static int getChunkIndex(int x, int z) { ++ return (x & 31) + ((z & 31) << 5); ++ } ++ ++ private static int getTimestamp() { ++ return (int) (System.currentTimeMillis() / 1000L); ++ } ++ ++ public void flush() throws IOException { ++ flushWrapper(); // sync ++ } ++ ++ public void flushWrapper() { ++ try { ++ save(); ++ } catch (IOException e) { ++ LOGGER.error("Failed to flush region file {}", path.toAbsolutePath(), e); ++ } ++ } ++ ++ public boolean doesChunkExist(ChunkPos pos) throws Exception { ++ throw new Exception("doesChunkExist is a stub"); ++ } ++ ++ private synchronized void save() throws IOException { ++ long timestamp = getTimestamp(); ++ short chunkCount = 0; ++ ++ File tempFile = new File(path.toString() + ".tmp"); ++ ++ try (FileOutputStream fileStream = new FileOutputStream(tempFile); ++ ByteArrayOutputStream zstdByteArray = new ByteArrayOutputStream(); ++ ZstdOutputStream zstdStream = new ZstdOutputStream(zstdByteArray, this.compressionLevel); ++ DataOutputStream zstdDataStream = new DataOutputStream(zstdStream); ++ DataOutputStream dataStream = new DataOutputStream(fileStream)) { ++ ++ dataStream.writeLong(SUPERBLOCK); ++ dataStream.writeByte(VERSION); ++ dataStream.writeLong(timestamp); ++ dataStream.writeByte(this.compressionLevel); ++ ++ ArrayList byteBuffers = new ArrayList<>(); ++ for (int i = 0; i < 1024; i++) { ++ if (this.bufferUncompressedSize[i] != 0) { ++ chunkCount += 1; ++ byte[] content = new byte[bufferUncompressedSize[i]]; ++ this.decompressor.decompress(buffer[i], 0, content, 0, bufferUncompressedSize[i]); ++ ++ byteBuffers.add(content); ++ } else byteBuffers.add(null); ++ } ++ for (int i = 0; i < 1024; i++) { ++ zstdDataStream.writeInt(this.bufferUncompressedSize[i]); // Write uncompressed size ++ zstdDataStream.writeInt(this.chunkTimestamps[i]); // Write timestamp ++ } ++ for (int i = 0; i < 1024; i++) { ++ if (byteBuffers.get(i) != null) ++ zstdDataStream.write(byteBuffers.get(i), 0, byteBuffers.get(i).length); ++ } ++ zstdDataStream.close(); ++ ++ dataStream.writeShort(chunkCount); ++ ++ byte[] compressed = zstdByteArray.toByteArray(); ++ ++ dataStream.writeInt(compressed.length); ++ dataStream.writeLong(0); ++ ++ dataStream.write(compressed, 0, compressed.length); ++ dataStream.writeLong(SUPERBLOCK); ++ ++ dataStream.flush(); ++ fileStream.getFD().sync(); ++ fileStream.getChannel().force(true); // Ensure atomicity on Btrfs ++ } ++ Files.move(tempFile.toPath(), this.path, StandardCopyOption.REPLACE_EXISTING); ++ this.lastFlushed = System.nanoTime(); ++ } ++ ++ public synchronized void write(ChunkPos pos, ByteBuffer buffer) { ++ try { ++ byte[] b = toByteArray(new ByteArrayInputStream(buffer.array())); ++ int uncompressedSize = b.length; ++ ++ int maxCompressedLength = this.compressor.maxCompressedLength(b.length); ++ byte[] compressed = new byte[maxCompressedLength]; ++ int compressedLength = this.compressor.compress(b, 0, b.length, compressed, 0, maxCompressedLength); ++ b = new byte[compressedLength]; ++ System.arraycopy(compressed, 0, b, 0, compressedLength); ++ ++ int index = getChunkIndex(pos.x, pos.z); ++ this.buffer[index] = b; ++ this.chunkTimestamps[index] = getTimestamp(); ++ this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] = uncompressedSize; ++ } catch (IOException e) { ++ LOGGER.error("Chunk write IOException {} {}", e, this.path); ++ } ++ ++ if ((System.nanoTime() - this.lastFlushed) >= TimeUnit.NANOSECONDS.toSeconds(RegionFormatConfig.linearFlushFrequency)) { ++ this.flushWrapper(); ++ } ++ } ++ ++ public DataOutputStream getChunkDataOutputStream(ChunkPos pos) { ++ return new DataOutputStream(new BufferedOutputStream(new ChunkBuffer(pos))); ++ } ++ ++ private byte[] toByteArray(InputStream in) throws IOException { ++ ByteArrayOutputStream out = new ByteArrayOutputStream(); ++ byte[] tempBuffer = new byte[4096]; ++ ++ int length; ++ while ((length = in.read(tempBuffer)) >= 0) { ++ out.write(tempBuffer, 0, length); ++ } ++ ++ return out.toByteArray(); ++ } ++ ++ @Nullable ++ public synchronized DataInputStream getChunkDataInputStream(ChunkPos pos) { ++ if (this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] != 0) { ++ byte[] content = new byte[bufferUncompressedSize[getChunkIndex(pos.x, pos.z)]]; ++ this.decompressor.decompress(this.buffer[getChunkIndex(pos.x, pos.z)], 0, content, 0, bufferUncompressedSize[getChunkIndex(pos.x, pos.z)]); ++ return new DataInputStream(new ByteArrayInputStream(content)); ++ } ++ return null; ++ } ++ ++ public void clear(ChunkPos pos) { ++ int i = getChunkIndex(pos.x, pos.z); ++ this.buffer[i] = null; ++ this.bufferUncompressedSize[i] = 0; ++ this.chunkTimestamps[i] = getTimestamp(); ++ this.flushWrapper(); ++ } ++ ++ public Path getPath() { ++ return this.path; ++ } ++ ++ public boolean hasChunk(ChunkPos pos) { ++ return this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] > 0; ++ } ++ ++ public void close() throws IOException { ++ if (closed) return; ++ closed = true; ++ flush(); // sync ++ } ++ ++ public void setOversized(int x, int z, boolean something) { ++ } ++ ++ public CompoundTag getOversizedData(int x, int z) throws IOException { ++ throw new IOException("getOversizedData is a stub " + this.path); ++ } ++ ++ public boolean isOversized(int x, int z) { ++ return false; ++ } ++ ++ private class ChunkBuffer extends ByteArrayOutputStream { ++ private final ChunkPos pos; ++ ++ public ChunkBuffer(ChunkPos chunkcoordintpair) { ++ super(); ++ this.pos = chunkcoordintpair; ++ } ++ ++ public void close() { ++ ByteBuffer bytebuffer = ByteBuffer.wrap(this.buf, 0, this.count); ++ LinearRegionFile.this.write(this.pos, bytebuffer); ++ } ++ } ++} +\ No newline at end of file