From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: NONPLAYT <76615486+NONPLAYT@users.noreply.github.com> Date: Sat, 22 Jun 2024 15:15:46 +0300 Subject: [PATCH] Implement Linear region format diff --git a/build.gradle.kts b/build.gradle.kts index dffee3fdef02135233bc9a915eebbb714830b889..4c65a8d8fed430b505fbaa28a3bb756a2b472776 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -30,6 +30,10 @@ dependencies { alsoShade(log4jPlugins.output) implementation("io.netty:netty-codec-haproxy:4.1.97.Final") // Paper - Add support for proxy protocol // Paper end + // DivineMC start + implementation("com.github.luben:zstd-jni:1.5.6-3") + implementation("org.lz4:lz4-java:1.8.0") + // DivineMC end implementation("org.apache.logging.log4j:log4j-iostreams:2.22.1") // Paper - remove exclusion implementation("org.ow2.asm:asm-commons:9.7") implementation("org.spongepowered:configurate-yaml:4.2.0-SNAPSHOT") // Paper - config files diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java index 73df26b27146bbad2106d57b22dd3c792ed3dd1d..a4d16996fae07f943ee078ce3d2e7b22747fc2d1 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java @@ -7,8 +7,8 @@ public interface ChunkSystemRegionFileStorage { public boolean moonrise$doesRegionFileNotExistNoIO(final int chunkX, final int chunkZ); - public RegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ); + public space.bxteam.divinemc.region.AbstractRegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ); // DivineMC - public RegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException; + public space.bxteam.divinemc.region.AbstractRegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException; // DivineMC } diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/RegionFileIOThread.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/RegionFileIOThread.java index c833f78d083b8f661087471c35bc90f65af1b525..cdc2bc217b19765c7a732b285138c2dc60f3f456 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/RegionFileIOThread.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/RegionFileIOThread.java @@ -1042,9 +1042,9 @@ public final class RegionFileIOThread extends PrioritisedQueueExecutorThread { return ((ChunkSystemRegionFileStorage)(Object)this.getCache()).moonrise$doesRegionFileNotExistNoIO(chunkX, chunkZ); } - public T computeForRegionFile(final int chunkX, final int chunkZ, final boolean existingOnly, final Function function) { + public T computeForRegionFile(final int chunkX, final int chunkZ, final boolean existingOnly, final Function function) { // DivineMC final RegionFileStorage cache = this.getCache(); - final RegionFile regionFile; + final space.bxteam.divinemc.region.AbstractRegionFile regionFile; // DivineMC synchronized (cache) { try { if (existingOnly) { @@ -1060,9 +1060,9 @@ public final class RegionFileIOThread extends PrioritisedQueueExecutorThread { } } - public T computeForRegionFileIfLoaded(final int chunkX, final int chunkZ, final Function function) { + public T computeForRegionFileIfLoaded(final int chunkX, final int chunkZ, final Function function) { // DivineMC final RegionFileStorage cache = this.getCache(); - final RegionFile regionFile; + final space.bxteam.divinemc.region.AbstractRegionFile regionFile; // DivineMC synchronized (cache) { regionFile = ((ChunkSystemRegionFileStorage)(Object)cache).moonrise$getRegionFileIfLoaded(chunkX, chunkZ); diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java index 3df813c086c2a8dfc29c08cb884c099ee698f412..254de097b4a4590d12c16c68d98748a3c9736938 100644 --- a/src/main/java/net/minecraft/server/MinecraftServer.java +++ b/src/main/java/net/minecraft/server/MinecraftServer.java @@ -915,10 +915,10 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop> progressMap = Reference2FloatMaps.synchronize(new Reference2FloatOpenHashMap()); volatile Component status = Component.translatable("optimizeWorld.stage.counting"); - static final Pattern REGEX = Pattern.compile("^r\\.(-?[0-9]+)\\.(-?[0-9]+)\\.mca$"); + static final Pattern REGEX = Pattern.compile("^r\\.(-?[0-9]+)\\.(-?[0-9]+)\\.(linear | mca)$"); // DivineMC final DimensionDataStorage overworldDataStorage; public WorldUpgrader(LevelStorageSource.LevelStorageAccess session, DataFixer dataFixer, RegistryAccess dynamicRegistryManager, boolean eraseCache, boolean recreateRegionFiles) { @@ -394,7 +394,7 @@ public class WorldUpgrader { private static List getAllChunkPositions(RegionStorageInfo key, Path regionDirectory) { File[] afile = regionDirectory.toFile().listFiles((file, s) -> { - return s.endsWith(".mca"); + return s.endsWith(".linear") || s.endsWith(".mca"); // DivineMC }); if (afile == null) { @@ -414,7 +414,7 @@ public class WorldUpgrader { List list1 = Lists.newArrayList(); try { - RegionFile regionfile = new RegionFile(key, file.toPath(), regionDirectory, true); + space.bxteam.divinemc.region.AbstractRegionFile regionfile = space.bxteam.divinemc.region.AbstractRegionFileFactory.getAbstractRegionFile(key, file.toPath(), regionDirectory, true); // DivineMC try { for (int i1 = 0; i1 < 32; ++i1) { @@ -477,7 +477,7 @@ public class WorldUpgrader { protected abstract boolean tryProcessOnePosition(T storage, ChunkPos chunkPos, ResourceKey worldKey); - private void onFileFinished(RegionFile regionFile) { + private void onFileFinished(space.bxteam.divinemc.region.AbstractRegionFile regionFile) { if (WorldUpgrader.this.recreateRegionFiles) { if (this.previousWriteFuture != null) { this.previousWriteFuture.join(); @@ -502,7 +502,7 @@ public class WorldUpgrader { } } - static record FileToUpgrade(RegionFile file, List chunksToUpgrade) { + static record FileToUpgrade(space.bxteam.divinemc.region.AbstractRegionFile file, List chunksToUpgrade) { // DivineMC } diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java index e761b63eebc1e76b2bb1cb887d83d0b63ad6ec90..add6311e12bd74c336bb9592e75493b5d8624824 100644 --- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java +++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java @@ -28,7 +28,7 @@ import net.minecraft.nbt.NbtIo; // Paper import net.minecraft.world.level.ChunkPos; import org.slf4j.Logger; -public class RegionFile implements AutoCloseable { +public class RegionFile implements AutoCloseable, space.bxteam.divinemc.region.AbstractRegionFile { // DivineMC private static final Logger LOGGER = LogUtils.getLogger(); private static final int SECTOR_BYTES = 4096; @@ -465,10 +465,10 @@ public class RegionFile implements AutoCloseable { private static int getChunkIndex(int x, int z) { return (x & 31) + (z & 31) * 32; } - synchronized boolean isOversized(int x, int z) { + public synchronized boolean isOversized(int x, int z) { // DivineMC return this.oversized[getChunkIndex(x, z)] == 1; } - synchronized void setOversized(int x, int z, boolean oversized) throws IOException { + public synchronized void setOversized(int x, int z, boolean oversized) throws IOException { // DivineMC final int offset = getChunkIndex(x, z); boolean previous = this.oversized[offset] == 1; this.oversized[offset] = (byte) (oversized ? 1 : 0); @@ -507,7 +507,7 @@ public class RegionFile implements AutoCloseable { return this.path.getParent().resolve(this.path.getFileName().toString().replaceAll("\\.mca$", "") + "_oversized_" + x + "_" + z + ".nbt"); } - synchronized CompoundTag getOversizedData(int x, int z) throws IOException { + public synchronized CompoundTag getOversizedData(int x, int z) throws IOException { // DivineMC Path file = getOversizedFile(x, z); try (DataInputStream out = new DataInputStream(new java.io.BufferedInputStream(new InflaterInputStream(Files.newInputStream(file))))) { return NbtIo.read((java.io.DataInput) out); diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java index 64b2608ae81373acb67e3e0453c61822c0d03087..5e28f5e72e8c372a0a70062364139b6f6790492d 100644 --- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java +++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java @@ -21,7 +21,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise public static final String ANVIL_EXTENSION = ".mca"; private static final int MAX_CACHE_SIZE = 256; - public final Long2ObjectLinkedOpenHashMap regionCache = new Long2ObjectLinkedOpenHashMap(); + public final Long2ObjectLinkedOpenHashMap regionCache = new Long2ObjectLinkedOpenHashMap(); // DivineMC private final RegionStorageInfo info; private final Path folder; private final boolean sync; @@ -30,9 +30,13 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise private static final int REGION_SHIFT = 5; private static final int MAX_NON_EXISTING_CACHE = 1024 * 64; private final it.unimi.dsi.fastutil.longs.LongLinkedOpenHashSet nonExistingRegionFiles = new it.unimi.dsi.fastutil.longs.LongLinkedOpenHashSet(MAX_NON_EXISTING_CACHE+1); - private static String getRegionFileName(final int chunkX, final int chunkZ) { + // DivineMC start - Linear region format + private static String getRegionFileName(final RegionStorageInfo info, final int chunkX, final int chunkZ) { + if (info.regionFormat().equals(space.bxteam.divinemc.region.RegionFileFormat.LINEAR)) + return "r." + (chunkX >> REGION_SHIFT) + "." + (chunkZ >> REGION_SHIFT) + ".linear"; return "r." + (chunkX >> REGION_SHIFT) + "." + (chunkZ >> REGION_SHIFT) + ".mca"; } + // DivineMC end private boolean doesRegionFilePossiblyExist(final long position) { synchronized (this.nonExistingRegionFiles) { @@ -66,15 +70,15 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise } @Override - public synchronized final RegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ) { + public synchronized final space.bxteam.divinemc.region.AbstractRegionFile moonrise$getRegionFileIfLoaded(final int chunkX, final int chunkZ) { return this.regionCache.getAndMoveToFirst(ChunkPos.asLong(chunkX >> REGION_SHIFT, chunkZ >> REGION_SHIFT)); } @Override - public synchronized final RegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException { + public synchronized final space.bxteam.divinemc.region.AbstractRegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException { final long key = ChunkPos.asLong(chunkX >> REGION_SHIFT, chunkZ >> REGION_SHIFT); - RegionFile ret = this.regionCache.getAndMoveToFirst(key); + space.bxteam.divinemc.region.AbstractRegionFile ret = this.regionCache.getAndMoveToFirst(key); if (ret != null) { return ret; } @@ -87,7 +91,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise this.regionCache.removeLast().close(); } - final Path regionPath = this.folder.resolve(getRegionFileName(chunkX, chunkZ)); + final Path regionPath = this.folder.resolve(getRegionFileName(this.info, chunkX, chunkZ)); // DivineMC if (!java.nio.file.Files.exists(regionPath)) { this.markNonExisting(key); @@ -98,7 +102,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise FileUtil.createDirectoriesSafe(this.folder); - ret = new RegionFile(this.info, regionPath, this.folder, this.sync); + ret = space.bxteam.divinemc.region.AbstractRegionFileFactory.getAbstractRegionFile(this.info, regionPath, this.folder, this.sync); // DivineMC this.regionCache.putAndMoveToFirst(key, ret); @@ -112,7 +116,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise this.info = storageKey; } - public RegionFile getRegionFile(ChunkPos chunkcoordintpair, boolean existingOnly) throws IOException { // CraftBukkit // Paper - public + public space.bxteam.divinemc.region.AbstractRegionFile getRegionFile(ChunkPos chunkcoordintpair, boolean existingOnly) throws IOException { // CraftBukkit // Paper - public // DivineMC // Paper start - rewrite chunk system if (existingOnly) { return this.moonrise$getRegionFileIfExists(chunkcoordintpair.x, chunkcoordintpair.z); @@ -120,7 +124,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise synchronized (this) { final long key = ChunkPos.asLong(chunkcoordintpair.x >> REGION_SHIFT, chunkcoordintpair.z >> REGION_SHIFT); - RegionFile ret = this.regionCache.getAndMoveToFirst(key); + space.bxteam.divinemc.region.AbstractRegionFile ret = this.regionCache.getAndMoveToFirst(key); // DivineMC if (ret != null) { return ret; } @@ -129,13 +133,13 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise this.regionCache.removeLast().close(); } - final Path regionPath = this.folder.resolve(getRegionFileName(chunkcoordintpair.x, chunkcoordintpair.z)); + final Path regionPath = this.folder.resolve(getRegionFileName(this.info, chunkcoordintpair.x, chunkcoordintpair.z)); // DivineMC this.createRegionFile(key); FileUtil.createDirectoriesSafe(this.folder); - ret = new RegionFile(this.info, regionPath, this.folder, this.sync); + ret = space.bxteam.divinemc.region.AbstractRegionFileFactory.getAbstractRegionFile(this.info, regionPath, this.folder, this.sync); // DivineMC this.regionCache.putAndMoveToFirst(key, ret); @@ -149,7 +153,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise org.apache.logging.log4j.LogManager.getLogger().fatal(msg + " (" + file.toString().replaceAll(".+[\\\\/]", "") + " - " + x + "," + z + ") Go clean it up to remove this message. /minecraft:tp " + (x<<4)+" 128 "+(z<<4) + " - DO NOT REPORT THIS TO DIVINEMC - You may ask for help on Discord, but do not file an issue. These error messages can not be removed."); } - private static CompoundTag readOversizedChunk(RegionFile regionfile, ChunkPos chunkCoordinate) throws IOException { + private static CompoundTag readOversizedChunk(space.bxteam.divinemc.region.AbstractRegionFile regionfile, ChunkPos chunkCoordinate) throws IOException { // DivineMC synchronized (regionfile) { try (DataInputStream datainputstream = regionfile.getChunkDataInputStream(chunkCoordinate)) { CompoundTag oversizedData = regionfile.getOversizedData(chunkCoordinate.x, chunkCoordinate.z); @@ -184,7 +188,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise @Nullable public CompoundTag read(ChunkPos pos) throws IOException { // CraftBukkit start - SPIGOT-5680: There's no good reason to preemptively create files on read, save that for writing - RegionFile regionfile = this.getRegionFile(pos, true); + space.bxteam.divinemc.region.AbstractRegionFile regionfile = this.getRegionFile(pos, true); // DivineMC if (regionfile == null) { return null; } @@ -235,7 +239,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise public void scanChunk(ChunkPos chunkPos, StreamTagVisitor scanner) throws IOException { // CraftBukkit start - SPIGOT-5680: There's no good reason to preemptively create files on read, save that for writing - RegionFile regionfile = this.getRegionFile(chunkPos, true); + space.bxteam.divinemc.region.AbstractRegionFile regionfile = this.getRegionFile(chunkPos, true); // DivineMC if (regionfile == null) { return; } @@ -265,7 +269,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise } public void write(ChunkPos pos, @Nullable CompoundTag nbt) throws IOException { // Paper - public - RegionFile regionfile = this.getRegionFile(pos, nbt == null); // CraftBukkit // Paper - rewrite chunk system + space.bxteam.divinemc.region.AbstractRegionFile regionfile = this.getRegionFile(pos, nbt == null); // CraftBukkit // Paper - rewrite chunk system // DivineMC // Paper start - rewrite chunk system if (regionfile == null) { // if the RegionFile doesn't exist, no point in deleting from it @@ -324,7 +328,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise // Paper start - rewrite chunk system synchronized (this) { final ExceptionCollector exceptionCollector = new ExceptionCollector<>(); - for (final RegionFile regionFile : this.regionCache.values()) { + for (final space.bxteam.divinemc.region.AbstractRegionFile regionFile : this.regionCache.values()) { // DivineMC try { regionFile.close(); } catch (final IOException ex) { @@ -341,7 +345,7 @@ public class RegionFileStorage implements AutoCloseable, ca.spottedleaf.moonrise // Paper start - rewrite chunk system synchronized (this) { final ExceptionCollector exceptionCollector = new ExceptionCollector<>(); - for (final RegionFile regionFile : this.regionCache.values()) { + for (final space.bxteam.divinemc.region.AbstractRegionFile regionFile : this.regionCache.values()) { // DivineMC try { regionFile.flush(); } catch (final IOException ex) { diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionStorageInfo.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionStorageInfo.java index 6111631c6673948b266286894603cc5e30451b02..b3d8d46ae1e5d4f6309ac85ac0adad8a03bbde16 100644 --- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionStorageInfo.java +++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionStorageInfo.java @@ -2,9 +2,27 @@ package net.minecraft.world.level.chunk.storage; import net.minecraft.resources.ResourceKey; import net.minecraft.world.level.Level; +import org.bukkit.Bukkit; +import org.bukkit.craftbukkit.CraftWorld; public record RegionStorageInfo(String level, ResourceKey dimension, String type) { public RegionStorageInfo withTypeSuffix(String suffix) { return new RegionStorageInfo(this.level, this.dimension, this.type + suffix); } + + // DivineMC start + public space.bxteam.divinemc.region.RegionFileFormat regionFormat() { + return ((CraftWorld) Bukkit.getWorld(level)) + .getHandle() + .divinemcConfig + .regionFormatName; + } + + public int linearCompressionLevel() { + return ((CraftWorld) Bukkit.getWorld(level)) + .getHandle() + .divinemcConfig + .regionFormatLinearCompressionLevel; + } + // DivineMC end } diff --git a/src/main/java/space/bxteam/divinemc/configuration/DivineConfig.java b/src/main/java/space/bxteam/divinemc/configuration/DivineConfig.java index d8fdabba24db864671bb3e5ff5062a2a67703725..f13a8dc3f11924fff0c9ab3d84a02ac315819f78 100644 --- a/src/main/java/space/bxteam/divinemc/configuration/DivineConfig.java +++ b/src/main/java/space/bxteam/divinemc/configuration/DivineConfig.java @@ -185,4 +185,15 @@ public class DivineConfig { else Bukkit.getLogger().log(Level.INFO, "Using " + asyncPathfindingMaxThreads + " threads for Async Pathfinding"); } + + public static int linearFlushFrequency = 10; + public static int linearFlushThreads = 1; + private static void regionFormatSettings() { + linearFlushFrequency = getInt("region-format.linear.flush-frequency", linearFlushFrequency); + linearFlushThreads = getInt("region-format.linear.flush-max-threads", linearFlushThreads); + if (linearFlushThreads < 0) + linearFlushThreads = Math.max(Runtime.getRuntime().availableProcessors() + linearFlushThreads, 1); + else + linearFlushThreads = Math.max(linearFlushThreads, 1); + } } diff --git a/src/main/java/space/bxteam/divinemc/configuration/DivineWorldConfig.java b/src/main/java/space/bxteam/divinemc/configuration/DivineWorldConfig.java index d94c51ea18d299dd52b9a8521a9cdc0d95b79356..e2daa71b44421d1801117d1b6baeb1c6eba7b177 100644 --- a/src/main/java/space/bxteam/divinemc/configuration/DivineWorldConfig.java +++ b/src/main/java/space/bxteam/divinemc/configuration/DivineWorldConfig.java @@ -3,10 +3,12 @@ package space.bxteam.divinemc.configuration; import org.apache.commons.lang.BooleanUtils; import org.bukkit.World; import org.bukkit.configuration.ConfigurationSection; +import space.bxteam.divinemc.region.RegionFileFormat; import java.util.List; import java.util.Map; import java.util.function.Predicate; +import java.util.logging.Level; import static space.bxteam.divinemc.configuration.DivineConfig.log; @@ -106,4 +108,21 @@ public class DivineWorldConfig { private void suppressErrorsFromDirtyAttributes() { suppressErrorsFromDirtyAttributes = getBoolean("suppress-errors-from-dirty-attributes", suppressErrorsFromDirtyAttributes); } + + public RegionFileFormat regionFormatName = RegionFileFormat.ANVIL; + public int regionFormatLinearCompressionLevel = 1; + private void regionFormatSettings() { + regionFormatName = RegionFileFormat.fromString(getString("region-format.format", regionFormatName.name())); + if (regionFormatName.equals(RegionFileFormat.INVALID)) { + log(Level.SEVERE, "Unknown region format in divinemc.yml: " + regionFormatName); + log(Level.SEVERE, "Falling back to ANVIL region file format."); + regionFormatName = RegionFileFormat.ANVIL; + } + regionFormatLinearCompressionLevel = getInt("region-format.linear.compression-level", regionFormatLinearCompressionLevel); + if (regionFormatLinearCompressionLevel > 23 || regionFormatLinearCompressionLevel < 1) { + log(Level.SEVERE, "Linear region compression level should be between 1 and 22 in divinemc.yml: " + regionFormatLinearCompressionLevel); + log(Level.SEVERE, "Falling back to compression level 1."); + regionFormatLinearCompressionLevel = 1; + } + } } diff --git a/src/main/java/space/bxteam/divinemc/region/AbstractRegionFile.java b/src/main/java/space/bxteam/divinemc/region/AbstractRegionFile.java new file mode 100644 index 0000000000000000000000000000000000000000..15ef37b50b153cfc89cfd8c0a5855dd5301765a4 --- /dev/null +++ b/src/main/java/space/bxteam/divinemc/region/AbstractRegionFile.java @@ -0,0 +1,25 @@ +package space.bxteam.divinemc.region; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.file.Path; +import net.minecraft.nbt.CompoundTag; +import net.minecraft.world.level.ChunkPos; + +public interface AbstractRegionFile { + + Path getPath(); + DataInputStream getChunkDataInputStream(ChunkPos pos) throws IOException; + DataOutputStream getChunkDataOutputStream(ChunkPos pos) throws IOException; + CompoundTag getOversizedData(int x, int z) throws IOException; + + boolean hasChunk(ChunkPos pos); + boolean doesChunkExist(ChunkPos pos); + boolean isOversized(int x, int z); + + void flush() throws IOException; + void close() throws IOException; + void clear(ChunkPos pos) throws IOException; + void setOversized(int x, int z, boolean oversized) throws IOException; +} diff --git a/src/main/java/space/bxteam/divinemc/region/AbstractRegionFileFactory.java b/src/main/java/space/bxteam/divinemc/region/AbstractRegionFileFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..8ea1d287ad76ba6e0505e61de1c8e81f62ec90f5 --- /dev/null +++ b/src/main/java/space/bxteam/divinemc/region/AbstractRegionFileFactory.java @@ -0,0 +1,21 @@ +package space.bxteam.divinemc.region; + +import java.io.IOException; +import java.nio.file.Path; +import net.minecraft.world.level.chunk.storage.RegionFile; +import net.minecraft.world.level.chunk.storage.RegionFileVersion; +import net.minecraft.world.level.chunk.storage.RegionStorageInfo; + +public class AbstractRegionFileFactory { + public static AbstractRegionFile getAbstractRegionFile(RegionStorageInfo storageKey, Path directory, Path path, boolean dsync) throws IOException { + return getAbstractRegionFile(storageKey, directory, path, RegionFileVersion.getCompressionFormat(), dsync); + } + + public static AbstractRegionFile getAbstractRegionFile(RegionStorageInfo storageKey, Path path, Path directory, RegionFileVersion compressionFormat, boolean dsync) throws IOException { + if (path.toString().endsWith(".linear")) { + return new LinearRegionFile(path, storageKey.linearCompressionLevel()); + } else { + return new RegionFile(storageKey, path, directory, compressionFormat, dsync); + } + } +} diff --git a/src/main/java/space/bxteam/divinemc/region/LinearRegionFile.java b/src/main/java/space/bxteam/divinemc/region/LinearRegionFile.java new file mode 100644 index 0000000000000000000000000000000000000000..e1bc33e6679c73bbcf26b14089bdb1ea001e6430 --- /dev/null +++ b/src/main/java/space/bxteam/divinemc/region/LinearRegionFile.java @@ -0,0 +1,300 @@ +package space.bxteam.divinemc.region; + +import com.github.luben.zstd.ZstdInputStream; +import com.github.luben.zstd.ZstdOutputStream; +import com.mojang.logging.LogUtils; +import java.io.BufferedOutputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.annotation.Nullable; +import net.jpountz.lz4.LZ4Compressor; +import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4FastDecompressor; +import net.minecraft.nbt.CompoundTag; +import net.minecraft.world.level.ChunkPos; +import org.slf4j.Logger; + +public class LinearRegionFile implements AbstractRegionFile, AutoCloseable { + + private static final long SUPERBLOCK = -4323716122432332390L; + private static final byte VERSION = 2; + private static final int HEADER_SIZE = 32; + private static final int FOOTER_SIZE = 8; + private static final Logger LOGGER = LogUtils.getLogger(); + private static final List SUPPORTED_VERSIONS = Arrays.asList((byte) 1, (byte) 2); + private static final LinearRegionFileFlusher linearRegionFileFlusher = new LinearRegionFileFlusher(); + private final byte[][] buffer = new byte[1024][]; + private final int[] bufferUncompressedSize = new int[1024]; + private final int[] chunkTimestamps = new int[1024]; + private final LZ4Compressor compressor; + private final LZ4FastDecompressor decompressor; + private final int compressionLevel; + private final AtomicBoolean markedToSave = new AtomicBoolean(false); + public boolean closed = false; + public Path path; + + + public LinearRegionFile(Path file, int compression) throws IOException { + this.path = file; + this.compressionLevel = compression; + this.compressor = LZ4Factory.fastestInstance().fastCompressor(); + this.decompressor = LZ4Factory.fastestInstance().fastDecompressor(); + + File regionFile = new File(this.path.toString()); + + Arrays.fill(this.bufferUncompressedSize, 0); + + if (!regionFile.canRead()) return; + + try (FileInputStream fileStream = new FileInputStream(regionFile); + DataInputStream rawDataStream = new DataInputStream(fileStream)) { + + long superBlock = rawDataStream.readLong(); + if (superBlock != SUPERBLOCK) + throw new RuntimeException("Invalid superblock: " + superBlock + " in " + file); + + byte version = rawDataStream.readByte(); + if (!SUPPORTED_VERSIONS.contains(version)) + throw new RuntimeException("Invalid version: " + version + " in " + file); + + // Skip newestTimestamp (Long) + Compression level (Byte) + Chunk count (Short): Unused. + rawDataStream.skipBytes(11); + + int dataCount = rawDataStream.readInt(); + long fileLength = file.toFile().length(); + if (fileLength != HEADER_SIZE + dataCount + FOOTER_SIZE) + throw new IOException("Invalid file length: " + this.path + " " + fileLength + " " + (HEADER_SIZE + dataCount + FOOTER_SIZE)); + + rawDataStream.skipBytes(8); // Skip data hash (Long): Unused. + + byte[] rawCompressed = new byte[dataCount]; + rawDataStream.readFully(rawCompressed, 0, dataCount); + + superBlock = rawDataStream.readLong(); + if (superBlock != SUPERBLOCK) + throw new IOException("Footer superblock invalid " + this.path); + + try (DataInputStream dataStream = new DataInputStream(new ZstdInputStream(new ByteArrayInputStream(rawCompressed)))) { + + int[] starts = new int[1024]; + for (int i = 0; i < 1024; i++) { + starts[i] = dataStream.readInt(); + dataStream.skipBytes(4); // Skip timestamps (Int): Unused. + } + + for (int i = 0; i < 1024; i++) { + if (starts[i] > 0) { + int size = starts[i]; + byte[] b = new byte[size]; + dataStream.readFully(b, 0, size); + + int maxCompressedLength = this.compressor.maxCompressedLength(size); + byte[] compressed = new byte[maxCompressedLength]; + int compressedLength = this.compressor.compress(b, 0, size, compressed, 0, maxCompressedLength); + b = new byte[compressedLength]; + System.arraycopy(compressed, 0, b, 0, compressedLength); + + this.buffer[i] = b; + this.bufferUncompressedSize[i] = size; + } + } + } + } + } + + private static int getChunkIndex(int x, int z) { + return (x & 31) + ((z & 31) << 5); + } + + private static int getTimestamp() { + return (int) (System.currentTimeMillis() / 1000L); + } + + public Path getPath() { + return this.path; + } + + public void flush() throws IOException { + if (isMarkedToSave()) flushWrapper(); // sync + } + + private void markToSave() { + linearRegionFileFlusher.scheduleSave(this); + markedToSave.set(true); + } + + public boolean isMarkedToSave() { + return markedToSave.getAndSet(false); + } + + public void flushWrapper() { + try { + save(); + } catch (IOException e) { + LOGGER.error("Failed to flush region file {}", path.toAbsolutePath(), e); + } + } + + public boolean doesChunkExist(ChunkPos pos) { + return false; + } + + private synchronized void save() throws IOException { + long timestamp = getTimestamp(); + short chunkCount = 0; + + File tempFile = new File(path.toString() + ".tmp"); + + try (FileOutputStream fileStream = new FileOutputStream(tempFile); + ByteArrayOutputStream zstdByteArray = new ByteArrayOutputStream(); + ZstdOutputStream zstdStream = new ZstdOutputStream(zstdByteArray, this.compressionLevel); + DataOutputStream zstdDataStream = new DataOutputStream(zstdStream); + DataOutputStream dataStream = new DataOutputStream(fileStream)) { + + dataStream.writeLong(SUPERBLOCK); + dataStream.writeByte(VERSION); + dataStream.writeLong(timestamp); + dataStream.writeByte(this.compressionLevel); + + ArrayList byteBuffers = new ArrayList<>(); + for (int i = 0; i < 1024; i++) { + if (this.bufferUncompressedSize[i] != 0) { + chunkCount += 1; + byte[] content = new byte[bufferUncompressedSize[i]]; + this.decompressor.decompress(buffer[i], 0, content, 0, bufferUncompressedSize[i]); + + byteBuffers.add(content); + } else byteBuffers.add(null); + } + for (int i = 0; i < 1024; i++) { + zstdDataStream.writeInt(this.bufferUncompressedSize[i]); // Write uncompressed size + zstdDataStream.writeInt(this.chunkTimestamps[i]); // Write timestamp + } + for (int i = 0; i < 1024; i++) { + if (byteBuffers.get(i) != null) + zstdDataStream.write(byteBuffers.get(i), 0, byteBuffers.get(i).length); + } + zstdDataStream.close(); + + dataStream.writeShort(chunkCount); + + byte[] compressed = zstdByteArray.toByteArray(); + + dataStream.writeInt(compressed.length); + dataStream.writeLong(0); + + dataStream.write(compressed, 0, compressed.length); + dataStream.writeLong(SUPERBLOCK); + + dataStream.flush(); + fileStream.getFD().sync(); + fileStream.getChannel().force(true); // Ensure atomicity on Btrfs + } + Files.move(tempFile.toPath(), this.path, StandardCopyOption.REPLACE_EXISTING); + } + + public synchronized void write(ChunkPos pos, ByteBuffer buffer) { + try { + byte[] b = toByteArray(new ByteArrayInputStream(buffer.array())); + int uncompressedSize = b.length; + + int maxCompressedLength = this.compressor.maxCompressedLength(b.length); + byte[] compressed = new byte[maxCompressedLength]; + int compressedLength = this.compressor.compress(b, 0, b.length, compressed, 0, maxCompressedLength); + b = new byte[compressedLength]; + System.arraycopy(compressed, 0, b, 0, compressedLength); + + int index = getChunkIndex(pos.x, pos.z); + this.buffer[index] = b; + this.chunkTimestamps[index] = getTimestamp(); + this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] = uncompressedSize; + } catch (IOException e) { + LOGGER.error("Chunk write IOException {} {}", e, this.path); + } + markToSave(); + } + + public DataOutputStream getChunkDataOutputStream(ChunkPos pos) { + return new DataOutputStream(new BufferedOutputStream(new ChunkBuffer(pos))); + } + + private byte[] toByteArray(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] tempBuffer = new byte[4096]; + + int length; + while ((length = in.read(tempBuffer)) >= 0) { + out.write(tempBuffer, 0, length); + } + + return out.toByteArray(); + } + + @Nullable + public synchronized DataInputStream getChunkDataInputStream(ChunkPos pos) { + if (this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] != 0) { + byte[] content = new byte[bufferUncompressedSize[getChunkIndex(pos.x, pos.z)]]; + this.decompressor.decompress(this.buffer[getChunkIndex(pos.x, pos.z)], 0, content, 0, bufferUncompressedSize[getChunkIndex(pos.x, pos.z)]); + return new DataInputStream(new ByteArrayInputStream(content)); + } + return null; + } + + public void clear(ChunkPos pos) { + int i = getChunkIndex(pos.x, pos.z); + this.buffer[i] = null; + this.bufferUncompressedSize[i] = 0; + this.chunkTimestamps[i] = getTimestamp(); + markToSave(); + } + + public boolean hasChunk(ChunkPos pos) { + return this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] > 0; + } + + public void close() throws IOException { + if (closed) return; + closed = true; + flush(); // sync + } + + public void setOversized(int x, int z, boolean something) { + } + + public CompoundTag getOversizedData(int x, int z) throws IOException { + throw new IOException("getOversizedData is a stub " + this.path); + } + + public boolean isOversized(int x, int z) { + return false; + } + + private class ChunkBuffer extends ByteArrayOutputStream { + private final ChunkPos pos; + + public ChunkBuffer(ChunkPos chunkcoordintpair) { + super(); + this.pos = chunkcoordintpair; + } + + public void close() { + ByteBuffer bytebuffer = ByteBuffer.wrap(this.buf, 0, this.count); + LinearRegionFile.this.write(this.pos, bytebuffer); + } + } +} \ No newline at end of file diff --git a/src/main/java/space/bxteam/divinemc/region/LinearRegionFileFlusher.java b/src/main/java/space/bxteam/divinemc/region/LinearRegionFileFlusher.java new file mode 100644 index 0000000000000000000000000000000000000000..2fba2a0004e689afc36ad1c9737d5b259a68ec45 --- /dev/null +++ b/src/main/java/space/bxteam/divinemc/region/LinearRegionFileFlusher.java @@ -0,0 +1,50 @@ +package space.bxteam.divinemc.region; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.Queue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import org.bukkit.Bukkit; +import space.bxteam.divinemc.configuration.DivineConfig; + +public class LinearRegionFileFlusher { + + private final Queue savingQueue = new LinkedBlockingQueue<>(); + private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder() + .setNameFormat("linear-flush-scheduler") + .build() + ); + private final ExecutorService executor = Executors.newFixedThreadPool( + DivineConfig.linearFlushThreads, + new ThreadFactoryBuilder() + .setNameFormat("linear-flusher-%d") + .build() + ); + + public LinearRegionFileFlusher() { + Bukkit.getLogger().info("Using " + DivineConfig.linearFlushThreads + " threads for linear region flushing."); + scheduler.scheduleAtFixedRate(this::pollAndFlush, 0L, DivineConfig.linearFlushFrequency, TimeUnit.SECONDS); + } + + public void scheduleSave(LinearRegionFile regionFile) { + if (savingQueue.contains(regionFile)) return; + savingQueue.add(regionFile); + } + + private void pollAndFlush() { + while (!savingQueue.isEmpty()) { + LinearRegionFile regionFile = savingQueue.poll(); + if (!regionFile.closed && regionFile.isMarkedToSave()) + executor.execute(regionFile::flushWrapper); + } + } + + public void shutdown() { + executor.shutdown(); + scheduler.shutdown(); + } +} \ No newline at end of file diff --git a/src/main/java/space/bxteam/divinemc/region/RegionFileFormat.java b/src/main/java/space/bxteam/divinemc/region/RegionFileFormat.java new file mode 100644 index 0000000000000000000000000000000000000000..a799c9b2bb262b86be3872ba2a920ca3e8cb9d02 --- /dev/null +++ b/src/main/java/space/bxteam/divinemc/region/RegionFileFormat.java @@ -0,0 +1,16 @@ +package space.bxteam.divinemc.region; + +public enum RegionFileFormat { + ANVIL, + LINEAR, + INVALID; + + public static RegionFileFormat fromString(String format) { + for (RegionFileFormat rff : values()) { + if (rff.name().equalsIgnoreCase(format)) { + return rff; + } + } + return RegionFileFormat.INVALID; + } +}