From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: MrHua269 Date: Mon, 1 Jul 2024 12:09:39 +0800 Subject: [PATCH] Added linear region format from LinearPurpur diff --git a/build.gradle.kts b/build.gradle.kts index 247598b6c73aca3743f4b16b47520f8ba16b2ed0..4d2651315d4d26de9ff7f4504d1e4e9a77fb9a89 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -37,6 +37,10 @@ dependencies { alsoShade(log4jPlugins.output) implementation("io.netty:netty-codec-haproxy:4.1.97.Final") // Paper - Add support for proxy protocol // Paper end + // LinearPurpur start - Linear format + implementation("com.github.luben:zstd-jni:1.5.6-2") + implementation("org.lz4:lz4-java:1.8.0") + // LinearPurpur end implementation("org.apache.logging.log4j:log4j-iostreams:2.19.0") // Paper - remove exclusion implementation("org.ow2.asm:asm-commons:9.7") implementation("org.spongepowered:configurate-yaml:4.2.0-SNAPSHOT") // Paper - config files diff --git a/src/main/java/io/papermc/paper/chunk/system/io/RegionFileIOThread.java b/src/main/java/io/papermc/paper/chunk/system/io/RegionFileIOThread.java index 2934f0cf0ef09c84739312b00186c2ef0019a165..e67543ef424d448096379bef118b8cb24b938964 100644 --- a/src/main/java/io/papermc/paper/chunk/system/io/RegionFileIOThread.java +++ b/src/main/java/io/papermc/paper/chunk/system/io/RegionFileIOThread.java @@ -816,7 +816,7 @@ public final class RegionFileIOThread extends PrioritisedQueueExecutorThread { final ChunkDataController taskController) { final ChunkPos chunkPos = new ChunkPos(chunkX, chunkZ); if (intendingToBlock) { - return taskController.computeForRegionFile(chunkX, chunkZ, true, (final RegionFile file) -> { + return taskController.computeForRegionFile(chunkX, chunkZ, true, (final org.purpurmc.purpur.region.AbstractRegionFile file) -> { // LinearPurpur if (file == null) { // null if no regionfile exists return Boolean.FALSE; } @@ -829,7 +829,7 @@ public final class RegionFileIOThread extends PrioritisedQueueExecutorThread { return Boolean.FALSE; } // else: it either exists or is not known, fall back to checking the loaded region file - return taskController.computeForRegionFileIfLoaded(chunkX, chunkZ, (final RegionFile file) -> { + return taskController.computeForRegionFileIfLoaded(chunkX, chunkZ, (final org.purpurmc.purpur.region.AbstractRegionFile file) -> { // LinearPurpur if (file == null) { // null if not loaded // not sure at this point, let the I/O thread figure it out return Boolean.TRUE; @@ -1131,9 +1131,9 @@ public final class RegionFileIOThread extends PrioritisedQueueExecutorThread { return this.getCache().doesRegionFileNotExistNoIO(new ChunkPos(chunkX, chunkZ)); } - public T computeForRegionFile(final int chunkX, final int chunkZ, final boolean existingOnly, final Function function) { + public T computeForRegionFile(final int chunkX, final int chunkZ, final boolean existingOnly, final Function function) { // LinearPurpur final RegionFileStorage cache = this.getCache(); - final RegionFile regionFile; + final org.purpurmc.purpur.region.AbstractRegionFile regionFile; // LinearPurpur synchronized (cache) { try { regionFile = cache.getRegionFile(new ChunkPos(chunkX, chunkZ), existingOnly, true); @@ -1146,19 +1146,19 @@ public final class RegionFileIOThread extends PrioritisedQueueExecutorThread { return function.apply(regionFile); } finally { if (regionFile != null) { - regionFile.fileLock.unlock(); + regionFile.getFileLock().unlock(); // LinearPurpur } } } - public T computeForRegionFileIfLoaded(final int chunkX, final int chunkZ, final Function function) { + public T computeForRegionFileIfLoaded(final int chunkX, final int chunkZ, final Function function) { // LinearPurpur final RegionFileStorage cache = this.getCache(); - final RegionFile regionFile; + final org.purpurmc.purpur.region.AbstractRegionFile regionFile; // LinearPurpur synchronized (cache) { regionFile = cache.getRegionFileIfLoaded(new ChunkPos(chunkX, chunkZ)); if (regionFile != null) { - regionFile.fileLock.lock(); + regionFile.getFileLock().lock(); // LinearPurpur } } @@ -1166,7 +1166,7 @@ public final class RegionFileIOThread extends PrioritisedQueueExecutorThread { return function.apply(regionFile); } finally { if (regionFile != null) { - regionFile.fileLock.unlock(); + regionFile.getFileLock().unlock(); // LinearPurpur } } } diff --git a/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java index 9017907c0ec67a37a506f09b7e4499cef7885279..f4313d6622f81154f1bdf0eef59ee9464816a2b4 100644 --- a/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java +++ b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java @@ -84,8 +84,15 @@ public class ThreadedWorldUpgrader { LOGGER.info("Found " + regionFiles.length + " regionfiles to convert"); LOGGER.info("Starting conversion now for world " + this.worldName); + // LinearPurpur start + org.purpurmc.purpur.region.RegionFileFormat formatName = me.earthme.luminol.config.modules.misc.RegionFormatConfig.regionFormatType; + int linearCompression = me.earthme.luminol.config.modules.misc.RegionFormatConfig.linearCompressionLevel; + boolean linearCrashOnBrokenSymlink = me.earthme.luminol.config.modules.misc.RegionFormatConfig.linearCrashOnBrokenSymlink; + LOGGER.info("Using format " + formatName + " (" + linearCompression + ")"); + // LinearPurpur end + final WorldInfo info = new WorldInfo(() -> worldPersistentData, - new ChunkStorage(regionFolder.toPath(), this.dataFixer, false), this.removeCaches, this.dimensionType, this.generatorKey); + new ChunkStorage(formatName, linearCompression, linearCrashOnBrokenSymlink, regionFolder.toPath(), this.dataFixer, false), this.removeCaches, this.dimensionType, this.generatorKey); // LinearPurpur long expectedChunks = (long)regionFiles.length * (32L * 32L); diff --git a/src/main/java/me/earthme/luminol/config/modules/misc/RegionFormatConfig.java b/src/main/java/me/earthme/luminol/config/modules/misc/RegionFormatConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..d2fd2a718c40360e9a648671c65630857903da12 --- /dev/null +++ b/src/main/java/me/earthme/luminol/config/modules/misc/RegionFormatConfig.java @@ -0,0 +1,52 @@ +package me.earthme.luminol.config.modules.misc; + +import com.electronwill.nightconfig.core.file.CommentedFileConfig; +import com.mojang.logging.LogUtils; +import me.earthme.luminol.config.ConfigInfo; +import me.earthme.luminol.config.DoNotLoad; +import me.earthme.luminol.config.EnumConfigCategory; +import me.earthme.luminol.config.IConfigModule; +import org.slf4j.Logger; +import org.purpurmc.purpur.region.RegionFileFormat; + +public class RegionFormatConfig implements IConfigModule { + @ConfigInfo(baseName = "region_format") + public static String regionFormatTypeName = "MCA"; + @DoNotLoad + public static RegionFileFormat regionFormatType = RegionFileFormat.LINEAR; + @ConfigInfo(baseName = "linear_compress_level") + public static int linearCompressionLevel = 1; + @ConfigInfo(baseName = "flush_every_seconds") + public static int linearFlushFrequency = 5; + @ConfigInfo(baseName = "linear_crash_on_broken_symlink") + public static boolean linearCrashOnBrokenSymlink = true; + + @DoNotLoad + private static final Logger logger = LogUtils.getLogger(); + + @Override + public EnumConfigCategory getCategory() { + return EnumConfigCategory.MISC; + } + + @Override + public String getBaseName() { + return "region_format_settings"; + } + + @Override + public void onLoaded(CommentedFileConfig configInstance) { + regionFormatType = RegionFileFormat.fromName(regionFormatTypeName); + if (regionFormatType == RegionFileFormat.UNKNOWN){ + logger.error("Unknown region file type {} !Falling back to MCA file.", regionFormatTypeName); + regionFormatType = RegionFileFormat.MCA; + } + + if (linearCompressionLevel > 23 || linearCompressionLevel < 1) { + logger.error("Linear region compression level should be between 1 and 22 in config: {}", linearCompressionLevel); + logger.error("Falling back to compression level 1."); + linearCompressionLevel = 1; + } + + } +} diff --git a/src/main/java/net/minecraft/server/level/ChunkMap.java b/src/main/java/net/minecraft/server/level/ChunkMap.java index 6ab9f83786dcfbd3156d2f2bd6da57baed1399f4..0e8023d7105ea9781482e26e75d35565c52bfcb7 100644 --- a/src/main/java/net/minecraft/server/level/ChunkMap.java +++ b/src/main/java/net/minecraft/server/level/ChunkMap.java @@ -211,7 +211,7 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider // Paper end - optimise chunk tick iteration public ChunkMap(ServerLevel world, LevelStorageSource.LevelStorageAccess session, DataFixer dataFixer, StructureTemplateManager structureTemplateManager, Executor executor, BlockableEventLoop mainThreadExecutor, LightChunkGetter chunkProvider, ChunkGenerator chunkGenerator, ChunkProgressListener worldGenerationProgressListener, ChunkStatusUpdateListener chunkStatusChangeListener, Supplier persistentStateManagerFactory, int viewDistance, boolean dsync) { - super(session.getDimensionPath(world.dimension()).resolve("region"), dataFixer, dsync); + super(me.earthme.luminol.config.modules.misc.RegionFormatConfig.regionFormatType, me.earthme.luminol.config.modules.misc.RegionFormatConfig.linearCompressionLevel, me.earthme.luminol.config.modules.misc.RegionFormatConfig.linearCrashOnBrokenSymlink, session.getDimensionPath(world.dimension()).resolve("region"), dataFixer, dsync); // LinearPurpur // Paper - rewrite chunk system this.tickingGenerated = new AtomicInteger(); //this.playerMap = new PlayerMap(); // Folia - region threading @@ -256,7 +256,7 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider this.lightEngine = new ThreadedLevelLightEngine(chunkProvider, this, this.level.dimensionType().hasSkyLight(), null, null); // Paper - rewrite chunk system this.distanceManager = new ChunkMap.ChunkDistanceManager(executor, mainThreadExecutor); this.overworldDataStorage = persistentStateManagerFactory; - this.poiManager = new PoiManager(path.resolve("poi"), dataFixer, dsync, iregistrycustom, world); + this.poiManager = new PoiManager(me.earthme.luminol.config.modules.misc.RegionFormatConfig.regionFormatType, me.earthme.luminol.config.modules.misc.RegionFormatConfig.linearCompressionLevel, me.earthme.luminol.config.modules.misc.RegionFormatConfig.linearCrashOnBrokenSymlink, path.resolve("poi"), dataFixer, dsync, iregistrycustom, world); // LinearPurpur this.setServerViewDistance(viewDistance); // Paper start this.dataRegionManager = new io.papermc.paper.chunk.SingleThreadChunkRegionManager(this.level, 2, (1.0 / 3.0), 1, 6, "Data", DataRegionData::new, DataRegionSectionData::new); @@ -808,13 +808,13 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider } public ChunkStatus getChunkStatusOnDiskIfCached(ChunkPos chunkPos) { - net.minecraft.world.level.chunk.storage.RegionFile regionFile = regionFileCache.getRegionFileIfLoaded(chunkPos); + org.purpurmc.purpur.region.AbstractRegionFile regionFile = regionFileCache.getRegionFileIfLoaded(chunkPos); // LinearPurpur return regionFile == null ? null : regionFile.getStatusIfCached(chunkPos.x, chunkPos.z); } public ChunkStatus getChunkStatusOnDisk(ChunkPos chunkPos) throws IOException { - net.minecraft.world.level.chunk.storage.RegionFile regionFile = regionFileCache.getRegionFile(chunkPos, true); + org.purpurmc.purpur.region.AbstractRegionFile regionFile = regionFileCache.getRegionFile(chunkPos, true); // LinearPurpur if (regionFile == null || !regionFileCache.chunkExists(chunkPos)) { return null; @@ -832,7 +832,7 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider } public void updateChunkStatusOnDisk(ChunkPos chunkPos, @Nullable CompoundTag compound) throws IOException { - net.minecraft.world.level.chunk.storage.RegionFile regionFile = regionFileCache.getRegionFile(chunkPos, false); + org.purpurmc.purpur.region.AbstractRegionFile regionFile = regionFileCache.getRegionFile(chunkPos, false); // LinearPurpur regionFile.setStatus(chunkPos.x, chunkPos.z, ChunkSerializer.getStatus(compound)); } diff --git a/src/main/java/net/minecraft/server/level/ServerLevel.java b/src/main/java/net/minecraft/server/level/ServerLevel.java index 43b8f64d7c14e6dd0975b24a3205806c4433f26f..6439c6974f0292d105581456fcb5e27fdfbae6d6 100644 --- a/src/main/java/net/minecraft/server/level/ServerLevel.java +++ b/src/main/java/net/minecraft/server/level/ServerLevel.java @@ -446,8 +446,8 @@ public class ServerLevel extends Level implements WorldGenLevel { private static final class EntityRegionFileStorage extends net.minecraft.world.level.chunk.storage.RegionFileStorage { - public EntityRegionFileStorage(Path directory, boolean dsync) { - super(directory, dsync); + public EntityRegionFileStorage(org.purpurmc.purpur.region.RegionFileFormat format, int linearCompression, boolean linearCrashOnBrokenSymlink, Path directory, boolean dsync) { // LinearPurpur + super(format, linearCompression, linearCrashOnBrokenSymlink, directory, dsync); // LinearPurpur } protected void write(ChunkPos pos, net.minecraft.nbt.CompoundTag nbt) throws IOException { @@ -808,7 +808,7 @@ public class ServerLevel extends Level implements WorldGenLevel { // CraftBukkit end boolean flag2 = minecraftserver.forceSynchronousWrites(); DataFixer datafixer = minecraftserver.getFixerUpper(); - this.entityStorage = new EntityRegionFileStorage(convertable_conversionsession.getDimensionPath(resourcekey).resolve("entities"), flag2); // Paper - rewrite chunk system //EntityPersistentStorage entitypersistentstorage = new EntityStorage(this, convertable_conversionsession.getDimensionPath(resourcekey).resolve("entities"), datafixer, flag2, minecraftserver); + this.entityStorage = new EntityRegionFileStorage(me.earthme.luminol.config.modules.misc.RegionFormatConfig.regionFormatType, me.earthme.luminol.config.modules.misc.RegionFormatConfig.linearCompressionLevel, me.earthme.luminol.config.modules.misc.RegionFormatConfig.linearCrashOnBrokenSymlink, convertable_conversionsession.getDimensionPath(resourcekey).resolve("entities"), flag2); // Paper - rewrite chunk system //EntityPersistentStorage entitypersistentstorage = new EntityStorage(this, convertable_conversionsession.getDimensionPath(resourcekey).resolve("entities"), datafixer, flag2, minecraftserver); // LinearPurpur // this.entityManager = new PersistentEntitySectionManager<>(Entity.class, new ServerLevel.EntityCallbacks(), entitypersistentstorage, this.entitySliceManager); // Paper // Paper - rewrite chunk system StructureTemplateManager structuretemplatemanager = minecraftserver.getStructureManager(); diff --git a/src/main/java/net/minecraft/util/worldupdate/WorldUpgrader.java b/src/main/java/net/minecraft/util/worldupdate/WorldUpgrader.java index 77dd632a266f4abed30b87b7909d77857c01e316..4938d86f62071f578822684f576b838296d4070f 100644 --- a/src/main/java/net/minecraft/util/worldupdate/WorldUpgrader.java +++ b/src/main/java/net/minecraft/util/worldupdate/WorldUpgrader.java @@ -61,7 +61,7 @@ public class WorldUpgrader { private volatile int skipped; private final Reference2FloatMap> progressMap = Reference2FloatMaps.synchronize(new Reference2FloatOpenHashMap()); private volatile Component status = Component.translatable("optimizeWorld.stage.counting"); - public static final Pattern REGEX = Pattern.compile("^r\\.(-?[0-9]+)\\.(-?[0-9]+)\\.mca$"); + public static Pattern REGEX = Pattern.compile("^r\\.(-?[0-9]+)\\.(-?[0-9]+)\\.(linear | mca)$"); // LinearPurpur private final DimensionDataStorage overworldDataStorage; public WorldUpgrader(LevelStorageSource.LevelStorageAccess session, DataFixer dataFixer, Registry dimensionOptionsRegistry, boolean eraseCache) { @@ -116,7 +116,13 @@ public class WorldUpgrader { ResourceKey resourcekey1 = (ResourceKey) iterator1.next(); Path path = this.levelStorage.getDimensionPath(resourcekey1); - builder1.put(resourcekey1, new ChunkStorage(path.resolve("region"), this.dataFixer, true)); + // LinearPurpur start + String worldName = this.levelStorage.getLevelId(); + org.purpurmc.purpur.region.RegionFileFormat formatName = ((org.bukkit.craftbukkit.CraftWorld) org.bukkit.Bukkit.getWorld(worldName)).getHandle().purpurConfig.regionFormatName; + int linearCompression = ((org.bukkit.craftbukkit.CraftWorld) org.bukkit.Bukkit.getWorld(worldName)).getHandle().purpurConfig.regionFormatLinearCompressionLevel; + boolean linearCrashOnBrokenSymlink = ((org.bukkit.craftbukkit.CraftWorld) org.bukkit.Bukkit.getWorld(worldName)).getHandle().purpurConfig.linearCrashOnBrokenSymlink; + builder1.put(resourcekey1, new ChunkStorage(formatName, linearCompression, linearCrashOnBrokenSymlink, path.resolve("region"), this.dataFixer, true)); + // LinearPurpur end } ImmutableMap, ChunkStorage> immutablemap1 = builder1.build(); @@ -241,7 +247,7 @@ public class WorldUpgrader { File file = this.levelStorage.getDimensionPath(world).toFile(); File file1 = new File(file, "region"); File[] afile = file1.listFiles((file2, s) -> { - return s.endsWith(".mca"); + return s.endsWith(".mca") || s.endsWith(".linear"); // LinearPurpur }); if (afile == null) { @@ -260,7 +266,11 @@ public class WorldUpgrader { int l = Integer.parseInt(matcher.group(2)) << 5; try { - RegionFile regionfile = new RegionFile(file2.toPath(), file1.toPath(), true); + // LinearPurpur start + String worldName = this.levelStorage.getLevelId(); + int linearCompression = ((org.bukkit.craftbukkit.CraftWorld) org.bukkit.Bukkit.getWorld(worldName)).getHandle().purpurConfig.regionFormatLinearCompressionLevel; + org.purpurmc.purpur.region.AbstractRegionFile regionfile = org.purpurmc.purpur.region.AbstractRegionFileFactory.getAbstractRegionFile(linearCompression, file2.toPath(), file1.toPath(), true); + // LinearPurpur end try { for (int i1 = 0; i1 < 32; ++i1) { diff --git a/src/main/java/net/minecraft/world/entity/ai/village/poi/PoiManager.java b/src/main/java/net/minecraft/world/entity/ai/village/poi/PoiManager.java index 79e9e5ece5859938ca0c86ead4c25cf5bde9da27..a26cc9b81d93f04cd741a2558f0787bb2037351e 100644 --- a/src/main/java/net/minecraft/world/entity/ai/village/poi/PoiManager.java +++ b/src/main/java/net/minecraft/world/entity/ai/village/poi/PoiManager.java @@ -59,8 +59,8 @@ public class PoiManager extends SectionStorage { // Paper end - rewrite chunk system - public PoiManager(Path path, DataFixer dataFixer, boolean dsync, RegistryAccess registryManager, LevelHeightAccessor world) { - super(path, PoiSection::codec, PoiSection::new, dataFixer, DataFixTypes.POI_CHUNK, dsync, registryManager, world); + public PoiManager(org.purpurmc.purpur.region.RegionFileFormat formatName, int linearCompression, boolean linearCrashOnBrokenSymlink, Path path, DataFixer dataFixer, boolean dsync, RegistryAccess registryManager, LevelHeightAccessor world) { // LinearPurpur + super(formatName, linearCompression, linearCrashOnBrokenSymlink, path, PoiSection::codec, PoiSection::new, dataFixer, DataFixTypes.POI_CHUNK, dsync, registryManager, world); // LinearPurpur this.world = (net.minecraft.server.level.ServerLevel)world; // Paper - rewrite chunk system } diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/ChunkStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/ChunkStorage.java index d16d7c2fed89fb1347df7ddd95856e7f08c22e8a..625a49f42bdd36772a8f4a992396f76822026911 100644 --- a/src/main/java/net/minecraft/world/level/chunk/storage/ChunkStorage.java +++ b/src/main/java/net/minecraft/world/level/chunk/storage/ChunkStorage.java @@ -36,9 +36,9 @@ public class ChunkStorage implements AutoCloseable { @Nullable private volatile LegacyStructureDataHandler legacyStructureHandler; - public ChunkStorage(Path directory, DataFixer dataFixer, boolean dsync) { + public ChunkStorage(org.purpurmc.purpur.region.RegionFileFormat format, int linearCompression, boolean linearCrashOnBrokenSymlink, Path directory, DataFixer dataFixer, boolean dsync) { // LinearPurpur this.fixerUpper = dataFixer; - this.regionFileCache = new RegionFileStorage(directory, dsync, true); // Paper - rewrite chunk system; async chunk IO & Attempt to recalculate regionfile header if it is corrupt + this.regionFileCache = new RegionFileStorage(format, linearCompression, linearCrashOnBrokenSymlink, directory, dsync, true); // Paper - rewrite chunk system; async chunk IO & Attempt to recalculate regionfile header if it is corrupt // LinearPurpur } public boolean isOldChunkAround(ChunkPos chunkPos, int checkRadius) { diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java index 6cf83502a954cce9c562ec036bfeddb477d38b73..7144ae7f8bfcac00b70bfe3c05af5f8ec824e46b 100644 --- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java +++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java @@ -26,7 +26,7 @@ import net.minecraft.nbt.NbtIo; // Paper import net.minecraft.world.level.ChunkPos; import org.slf4j.Logger; -public class RegionFile implements AutoCloseable { +public class RegionFile implements AutoCloseable, org.purpurmc.purpur.region.AbstractRegionFile { // LinearPurpur private static final Logger LOGGER = LogUtils.getLogger(); private static final int SECTOR_BYTES = 4096; @@ -50,6 +50,16 @@ public class RegionFile implements AutoCloseable { public final java.util.concurrent.locks.ReentrantLock fileLock = new java.util.concurrent.locks.ReentrantLock(); // Paper public final Path regionFile; // Paper + // LinearPurpur start - Abstract getters + public Path getRegionFile() { + return this.regionFile; + } + + public java.util.concurrent.locks.ReentrantLock getFileLock() { + return this.fileLock; + } + // LinearPurpur end + // Paper start - Attempt to recalculate regionfile header if it is corrupt private static long roundToSectors(long bytes) { long sectors = bytes >>> 12; // 4096 = 2^12 @@ -128,7 +138,7 @@ public class RegionFile implements AutoCloseable { } // note: only call for CHUNK regionfiles - boolean recalculateHeader() throws IOException { + public boolean recalculateHeader() throws IOException { // LinearPurpur if (!this.canRecalcHeader) { return false; } @@ -955,10 +965,10 @@ public class RegionFile implements AutoCloseable { private static int getChunkIndex(int x, int z) { return (x & 31) + (z & 31) * 32; } - synchronized boolean isOversized(int x, int z) { + public synchronized boolean isOversized(int x, int z) { // LinearPurpur return this.oversized[getChunkIndex(x, z)] == 1; } - synchronized void setOversized(int x, int z, boolean oversized) throws IOException { + public synchronized void setOversized(int x, int z, boolean oversized) throws IOException { // LinearPurpur final int offset = getChunkIndex(x, z); boolean previous = this.oversized[offset] == 1; this.oversized[offset] = (byte) (oversized ? 1 : 0); @@ -997,7 +1007,7 @@ public class RegionFile implements AutoCloseable { return this.regionFile.getParent().resolve(this.regionFile.getFileName().toString().replaceAll("\\.mca$", "") + "_oversized_" + x + "_" + z + ".nbt"); } - synchronized CompoundTag getOversizedData(int x, int z) throws IOException { + public synchronized CompoundTag getOversizedData(int x, int z) throws IOException { // LinearPurpur Path file = getOversizedFile(x, z); try (DataInputStream out = new DataInputStream(new java.io.BufferedInputStream(new InflaterInputStream(Files.newInputStream(file))))) { return NbtIo.read((java.io.DataInput) out); diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java index fe312b1aef579cb4bf81bdd967cf72ff880d7505..c96fbbb4a4d225df8d6969cac74767d13f845f00 100644 --- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java +++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java @@ -16,14 +16,21 @@ import net.minecraft.nbt.NbtIo; import net.minecraft.nbt.StreamTagVisitor; import net.minecraft.util.ExceptionCollector; import net.minecraft.world.level.ChunkPos; +import org.purpurmc.purpur.region.RegionFileFormat; public class RegionFileStorage implements AutoCloseable { + private static final org.slf4j.Logger LOGGER = com.mojang.logging.LogUtils.getLogger(); // LinearPurpur public static final String ANVIL_EXTENSION = ".mca"; private static final int MAX_CACHE_SIZE = 256; - public final Long2ObjectLinkedOpenHashMap regionCache = new Long2ObjectLinkedOpenHashMap(); + public final Long2ObjectLinkedOpenHashMap regionCache = new Long2ObjectLinkedOpenHashMap(); // LinearPurpur private final Path folder; private final boolean sync; + // LinearPurpur start - Per world chunk format + public final org.purpurmc.purpur.region.RegionFileFormat format; + public final int linearCompression; + public final boolean linearCrashOnBrokenSymlink; + // LinearPurpur end private final boolean isChunkData; // Paper // Paper start - cache regionfile does not exist state @@ -55,11 +62,16 @@ public class RegionFileStorage implements AutoCloseable { } // Paper end - cache regionfile does not exist state - protected RegionFileStorage(Path directory, boolean dsync) { // Paper - protected constructor + protected RegionFileStorage(org.purpurmc.purpur.region.RegionFileFormat format, int linearCompression, boolean linearCrashOnBrokenSymlink, Path directory, boolean dsync) { // Paper - protected constructor // LinearPurpur // Paper start - add isChunkData param - this(directory, dsync, false); + this(format, linearCompression, linearCrashOnBrokenSymlink, directory, dsync, false); } - RegionFileStorage(Path directory, boolean dsync, boolean isChunkData) { + RegionFileStorage(org.purpurmc.purpur.region.RegionFileFormat format, int linearCompression, boolean linearCrashOnBrokenSymlink, Path directory, boolean dsync, boolean isChunkData) { // LinearPurpur + // LinearPurpur start + this.format = format; + this.linearCompression = linearCompression; + this.linearCrashOnBrokenSymlink = linearCrashOnBrokenSymlink; + // LinearPurpur end this.isChunkData = isChunkData; // Paper end - add isChunkData param this.folder = directory; @@ -70,7 +82,7 @@ public class RegionFileStorage implements AutoCloseable { @Nullable public static ChunkPos getRegionFileCoordinates(Path file) { String fileName = file.getFileName().toString(); - if (!fileName.startsWith("r.") || !fileName.endsWith(".mca")) { + if (!fileName.startsWith("r.") || !fileName.endsWith(".mca") || !fileName.endsWith(".linear")) { // LinearPurpur return null; } @@ -90,29 +102,43 @@ public class RegionFileStorage implements AutoCloseable { } } - public synchronized RegionFile getRegionFileIfLoaded(ChunkPos chunkcoordintpair) { + public synchronized org.purpurmc.purpur.region.AbstractRegionFile getRegionFileIfLoaded(ChunkPos chunkcoordintpair) { // LinearPurpur return this.regionCache.getAndMoveToFirst(ChunkPos.asLong(chunkcoordintpair.getRegionX(), chunkcoordintpair.getRegionZ())); } public synchronized boolean chunkExists(ChunkPos pos) throws IOException { - RegionFile regionfile = getRegionFile(pos, true); + org.purpurmc.purpur.region.AbstractRegionFile regionfile = getRegionFile(pos, true); // LinearPurpur return regionfile != null ? regionfile.hasChunk(pos) : false; } - public synchronized RegionFile getRegionFile(ChunkPos chunkcoordintpair, boolean existingOnly) throws IOException { // CraftBukkit + // LinearPurpur start + private void guardAgainstBrokenSymlinks(Path path) throws IOException { + if (!linearCrashOnBrokenSymlink) return; + if (!(this.format == RegionFileFormat.LINEAR)) return; + if (!java.nio.file.Files.isSymbolicLink(path)) return; + Path link = java.nio.file.Files.readSymbolicLink(path); + if (!java.nio.file.Files.exists(link) || !java.nio.file.Files.isReadable(link)) { + LOGGER.error("Linear region file {} is a broken symbolic link, crashing to prevent data loss", path); + net.minecraft.server.MinecraftServer.getServer().halt(false); + throw new IOException("Linear region file " + path + " is a broken symbolic link, crashing to prevent data loss"); + } + } + // LinearPurpur end + + public synchronized org.purpurmc.purpur.region.AbstractRegionFile getRegionFile(ChunkPos chunkcoordintpair, boolean existingOnly) throws IOException { // CraftBukkit // LinearPurpur return this.getRegionFile(chunkcoordintpair, existingOnly, false); } - public synchronized RegionFile getRegionFile(ChunkPos chunkcoordintpair, boolean existingOnly, boolean lock) throws IOException { + public synchronized org.purpurmc.purpur.region.AbstractRegionFile getRegionFile(ChunkPos chunkcoordintpair, boolean existingOnly, boolean lock) throws IOException { // LinearPurpur // Paper end long i = ChunkPos.asLong(chunkcoordintpair.getRegionX(), chunkcoordintpair.getRegionZ()); final long regionPos = i; // Paper - OBFHELPER - RegionFile regionfile = (RegionFile) this.regionCache.getAndMoveToFirst(i); + org.purpurmc.purpur.region.AbstractRegionFile regionfile = this.regionCache.getAndMoveToFirst(i); // LinearPurpur if (regionfile != null) { // Paper start if (lock) { // must be in this synchronized block - regionfile.fileLock.lock(); + regionfile.getFileLock().lock(); // LinearPurpur } // Paper end return regionfile; @@ -123,28 +149,42 @@ public class RegionFileStorage implements AutoCloseable { } // Paper end - cache regionfile does not exist state if (this.regionCache.size() >= io.papermc.paper.configuration.GlobalConfiguration.get().misc.regionFileCacheSize) { // Paper - Sanitise RegionFileCache and make configurable - ((RegionFile) this.regionCache.removeLast()).close(); + this.regionCache.removeLast().close(); // LinearPurpur } // Paper - only create directory if not existing only - moved down Path path = this.folder; int j = chunkcoordintpair.getRegionX(); - Path path1 = path.resolve("r." + j + "." + chunkcoordintpair.getRegionZ() + ".mca"); // Paper - diff on change - if (existingOnly && !java.nio.file.Files.exists(path1)) { // Paper start - cache regionfile does not exist state - this.markNonExisting(regionPos); - return null; // CraftBukkit + // LinearPurpur start - Polyglot + Path path1; + if (existingOnly) { + Path anvil = path.resolve("r." + j + "." + chunkcoordintpair.getRegionZ() + ".mca"); + Path linear = path.resolve("r." + j + "." + chunkcoordintpair.getRegionZ() + ".linear"); + guardAgainstBrokenSymlinks(linear); + if (java.nio.file.Files.exists(anvil)) path1 = anvil; + else if (java.nio.file.Files.exists(linear)) path1 = linear; + else { + this.markNonExisting(regionPos); + return null; + } + // LinearPurpur end } else { + // LinearPurpur start - Polyglot + String extension = this.format.getExtensionName(); + path1 = path.resolve("r." + j + "." + chunkcoordintpair.getRegionZ() + extension); + // LinearPurpur end + guardAgainstBrokenSymlinks(path1); // LinearPurpur - Crash on broken symlink this.createRegionFile(regionPos); } // Paper end - cache regionfile does not exist state FileUtil.createDirectoriesSafe(this.folder); // Paper - only create directory if not existing only - moved from above - RegionFile regionfile1 = new RegionFile(path1, this.folder, this.sync, this.isChunkData); // Paper - allow for chunk regionfiles to regen header + org.purpurmc.purpur.region.AbstractRegionFile regionfile1 = org.purpurmc.purpur.region.AbstractRegionFileFactory.getAbstractRegionFile(this.linearCompression, path1, this.folder, this.sync, this.isChunkData); // Paper - allow for chunk regionfiles to regen header // LinearPurpur this.regionCache.putAndMoveToFirst(i, regionfile1); // Paper start if (lock) { // must be in this synchronized block - regionfile1.fileLock.lock(); + regionfile1.getFileLock().lock(); // LinearPurpur } // Paper end return regionfile1; @@ -156,7 +196,7 @@ public class RegionFileStorage implements AutoCloseable { org.apache.logging.log4j.LogManager.getLogger().fatal(msg + " (" + file.toString().replaceAll(".+[\\\\/]", "") + " - " + x + "," + z + ") Go clean it up to remove this message. /minecraft:tp " + (x<<4)+" 128 "+(z<<4) + " - DO NOT REPORT THIS TO PAPER - You may ask for help on Discord, but do not file an issue. These error messages can not be removed."); } - private static CompoundTag readOversizedChunk(RegionFile regionfile, ChunkPos chunkCoordinate) throws IOException { + private static CompoundTag readOversizedChunk(org.purpurmc.purpur.region.AbstractRegionFile regionfile, ChunkPos chunkCoordinate) throws IOException { // LinearPurpur synchronized (regionfile) { try (DataInputStream datainputstream = regionfile.getChunkDataInputStream(chunkCoordinate)) { CompoundTag oversizedData = regionfile.getOversizedData(chunkCoordinate.x, chunkCoordinate.z); @@ -191,14 +231,14 @@ public class RegionFileStorage implements AutoCloseable { @Nullable public CompoundTag read(ChunkPos pos) throws IOException { // CraftBukkit start - SPIGOT-5680: There's no good reason to preemptively create files on read, save that for writing - RegionFile regionfile = this.getRegionFile(pos, true, true); // Paper + org.purpurmc.purpur.region.AbstractRegionFile regionfile = this.getRegionFile(pos, true, true); // Paper // LinearPurpur if (regionfile == null) { return null; } // Paper start - Add regionfile parameter return this.read(pos, regionfile); } - public CompoundTag read(ChunkPos pos, RegionFile regionfile) throws IOException { + public CompoundTag read(ChunkPos pos, org.purpurmc.purpur.region.AbstractRegionFile regionfile) throws IOException { // LinearPurpur // We add the regionfile parameter to avoid the potential deadlock (on fileLock) if we went back to obtain a regionfile // if we decide to re-read // Paper end @@ -208,7 +248,7 @@ public class RegionFileStorage implements AutoCloseable { // Paper start if (regionfile.isOversized(pos.x, pos.z)) { - printOversizedLog("Loading Oversized Chunk!", regionfile.regionFile, pos.x, pos.z); + printOversizedLog("Loading Oversized Chunk!", regionfile.getRegionFile(), pos.x, pos.z); // LinearPurpur return readOversizedChunk(regionfile, pos); } // Paper end @@ -222,12 +262,12 @@ public class RegionFileStorage implements AutoCloseable { if (this.isChunkData) { ChunkPos chunkPos = ChunkSerializer.getChunkCoordinate(nbttagcompound); if (!chunkPos.equals(pos)) { - net.minecraft.server.MinecraftServer.LOGGER.error("Attempting to read chunk data at " + pos + " but got chunk data for " + chunkPos + " instead! Attempting regionfile recalculation for regionfile " + regionfile.regionFile.toAbsolutePath()); + net.minecraft.server.MinecraftServer.LOGGER.error("Attempting to read chunk data at " + pos + " but got chunk data for " + chunkPos + " instead! Attempting regionfile recalculation for regionfile " + regionfile.getRegionFile().toAbsolutePath()); // LinearPurpur if (regionfile.recalculateHeader()) { - regionfile.fileLock.lock(); // otherwise we will unlock twice and only lock once. + regionfile.getFileLock().lock(); // otherwise we will unlock twice and only lock once. // LinearPurpur return this.read(pos, regionfile); } - net.minecraft.server.MinecraftServer.LOGGER.error("Can't recalculate regionfile header, regenerating chunk " + pos + " for " + regionfile.regionFile.toAbsolutePath()); + net.minecraft.server.MinecraftServer.LOGGER.error("Can't recalculate regionfile header, regenerating chunk " + pos + " for " + regionfile.getRegionFile().toAbsolutePath()); // LinearPurpur return null; } } @@ -261,13 +301,13 @@ public class RegionFileStorage implements AutoCloseable { return nbttagcompound; } finally { // Paper start - regionfile.fileLock.unlock(); + regionfile.getFileLock().unlock(); // LinearPurpur } // Paper end } public void scanChunk(ChunkPos chunkPos, StreamTagVisitor scanner) throws IOException { // CraftBukkit start - SPIGOT-5680: There's no good reason to preemptively create files on read, save that for writing - RegionFile regionfile = this.getRegionFile(chunkPos, true); + org.purpurmc.purpur.region.AbstractRegionFile regionfile = this.getRegionFile(chunkPos, true); // LinearPurpur if (regionfile == null) { return; } @@ -298,7 +338,7 @@ public class RegionFileStorage implements AutoCloseable { protected void write(ChunkPos pos, @Nullable CompoundTag nbt) throws IOException { // Paper start - rewrite chunk system - RegionFile regionfile = this.getRegionFile(pos, nbt == null, true); // CraftBukkit + org.purpurmc.purpur.region.AbstractRegionFile regionfile = this.getRegionFile(pos, nbt == null, true); // CraftBukkit // LinearPurpur if (nbt == null && regionfile == null) { return; } @@ -353,7 +393,7 @@ public class RegionFileStorage implements AutoCloseable { // Paper end - Chunk save reattempt // Paper start - rewrite chunk system } finally { - regionfile.fileLock.unlock(); + regionfile.getFileLock().unlock(); // LinearPurpur } // Paper end - rewrite chunk system } @@ -363,7 +403,7 @@ public class RegionFileStorage implements AutoCloseable { ObjectIterator objectiterator = this.regionCache.values().iterator(); while (objectiterator.hasNext()) { - RegionFile regionfile = (RegionFile) objectiterator.next(); + org.purpurmc.purpur.region.AbstractRegionFile regionfile = (org.purpurmc.purpur.region.AbstractRegionFile) objectiterator.next(); // LinearPurpur try { regionfile.close(); @@ -379,7 +419,7 @@ public class RegionFileStorage implements AutoCloseable { ObjectIterator objectiterator = this.regionCache.values().iterator(); while (objectiterator.hasNext()) { - RegionFile regionfile = (RegionFile) objectiterator.next(); + org.purpurmc.purpur.region.AbstractRegionFile regionfile = (org.purpurmc.purpur.region.AbstractRegionFile) objectiterator.next(); // LinearPurpur regionfile.flush(); } diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/SectionStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/SectionStorage.java index 4ac5024936987c15f927e3148af4bfa57228ad1e..72cf6907e020f84a126cc108cc92e20669f74762 100644 --- a/src/main/java/net/minecraft/world/level/chunk/storage/SectionStorage.java +++ b/src/main/java/net/minecraft/world/level/chunk/storage/SectionStorage.java @@ -48,6 +48,9 @@ public class SectionStorage extends RegionFileStorage implements AutoCloseabl protected final LevelHeightAccessor levelHeightAccessor; public SectionStorage( + org.purpurmc.purpur.region.RegionFileFormat format, // LinearPurpur + int linearCompression, // LinearPurpur + boolean linearCrashOnBrokenSymlink, // LinearPurpur Path path, Function> codecFactory, Function factory, @@ -57,7 +60,7 @@ public class SectionStorage extends RegionFileStorage implements AutoCloseabl RegistryAccess dynamicRegistryManager, LevelHeightAccessor world ) { - super(path, dsync); // Paper - remove mojang I/O thread + super(format, linearCompression, linearCrashOnBrokenSymlink, path, dsync); // Paper - remove mojang I/O thread // LinearPurpur this.codec = codecFactory; this.factory = factory; this.fixerUpper = dataFixer; diff --git a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java index f325532b326fd7fe8c36003833890db57371eacc..b1681d6c9e8cb423e6027b9955adcff072435520 100644 --- a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java +++ b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java @@ -576,7 +576,7 @@ public class CraftWorld extends CraftRegionAccessor implements World { world.getChunk(x, z); // make sure we're at ticket level 32 or lower return true; } - net.minecraft.world.level.chunk.storage.RegionFile file; + org.purpurmc.purpur.region.AbstractRegionFile file; // LinearPurpur try { file = world.getChunkSource().chunkMap.regionFileCache.getRegionFile(chunkPos, false); } catch (java.io.IOException ex) { diff --git a/src/main/java/org/purpurmc/purpur/region/AbstractRegionFile.java b/src/main/java/org/purpurmc/purpur/region/AbstractRegionFile.java new file mode 100644 index 0000000000000000000000000000000000000000..e6ab1ca9a699c3f57855f2acc8d08b8a005bce2f --- /dev/null +++ b/src/main/java/org/purpurmc/purpur/region/AbstractRegionFile.java @@ -0,0 +1,31 @@ +package org.purpurmc.purpur.region; + +import net.minecraft.nbt.CompoundTag; +import net.minecraft.world.level.ChunkPos; +import net.minecraft.world.level.chunk.ChunkStatus; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.file.Path; +import java.util.concurrent.locks.ReentrantLock; + +public interface AbstractRegionFile { + void flush() throws IOException; + void clear(ChunkPos pos) throws IOException; + void close() throws IOException; + void setStatus(int x, int z, ChunkStatus status); + void setOversized(int x, int z, boolean b) throws IOException; + + boolean hasChunk(ChunkPos pos); + boolean doesChunkExist(ChunkPos pos) throws Exception; + boolean isOversized(int x, int z); + boolean recalculateHeader() throws IOException; + + DataOutputStream getChunkDataOutputStream(ChunkPos pos) throws IOException; + DataInputStream getChunkDataInputStream(ChunkPos pos) throws IOException; + CompoundTag getOversizedData(int x, int z) throws IOException; + ChunkStatus getStatusIfCached(int x, int z); + ReentrantLock getFileLock(); + Path getRegionFile(); +} diff --git a/src/main/java/org/purpurmc/purpur/region/AbstractRegionFileFactory.java b/src/main/java/org/purpurmc/purpur/region/AbstractRegionFileFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..c88ff6fda185a8489cbefa51a7b09ccba4f11461 --- /dev/null +++ b/src/main/java/org/purpurmc/purpur/region/AbstractRegionFileFactory.java @@ -0,0 +1,29 @@ +package org.purpurmc.purpur.region; + +import net.minecraft.world.level.chunk.storage.RegionFile; +import net.minecraft.world.level.chunk.storage.RegionFileVersion; + +import java.io.IOException; +import java.nio.file.Path; + +public class AbstractRegionFileFactory { + public static AbstractRegionFile getAbstractRegionFile(int linearCompression, Path file, Path directory, boolean dsync) throws IOException { + return getAbstractRegionFile(linearCompression, file, directory, RegionFileVersion.VERSION_DEFLATE, dsync); + } + + public static AbstractRegionFile getAbstractRegionFile(int linearCompression, Path file, Path directory, boolean dsync, boolean canRecalcHeader) throws IOException { + return getAbstractRegionFile(linearCompression, file, directory, RegionFileVersion.VERSION_DEFLATE, dsync, canRecalcHeader); + } + + public static AbstractRegionFile getAbstractRegionFile(int linearCompression, Path file, Path directory, RegionFileVersion outputChunkStreamVersion, boolean dsync) throws IOException { + return getAbstractRegionFile(linearCompression, file, directory, outputChunkStreamVersion, dsync, false); + } + + public static AbstractRegionFile getAbstractRegionFile(int linearCompression, Path file, Path directory, RegionFileVersion outputChunkStreamVersion, boolean dsync, boolean canRecalcHeader) throws IOException { + if (file.toString().endsWith(".linear")) { + return new LinearRegionFile(file, linearCompression); + } else { + return new RegionFile(file, directory, outputChunkStreamVersion, dsync, canRecalcHeader); + } + } +} diff --git a/src/main/java/org/purpurmc/purpur/region/LinearRegionFile.java b/src/main/java/org/purpurmc/purpur/region/LinearRegionFile.java new file mode 100644 index 0000000000000000000000000000000000000000..fe6c06a1a4b57d8da01fae8fa6a881f6f0496dd4 --- /dev/null +++ b/src/main/java/org/purpurmc/purpur/region/LinearRegionFile.java @@ -0,0 +1,314 @@ +package org.purpurmc.purpur.region; + +import com.github.luben.zstd.ZstdInputStream; +import com.github.luben.zstd.ZstdOutputStream; +import com.mojang.logging.LogUtils; +import net.jpountz.lz4.LZ4Compressor; +import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4FastDecompressor; +import net.minecraft.nbt.CompoundTag; +import net.minecraft.world.level.ChunkPos; +import net.minecraft.world.level.chunk.ChunkStatus; +import org.slf4j.Logger; + +import javax.annotation.Nullable; +import java.io.*; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantLock; + +public class LinearRegionFile implements AbstractRegionFile, AutoCloseable { + private static final long SUPERBLOCK = -4323716122432332390L; + private static final byte VERSION = 2; + private static final int HEADER_SIZE = 32; + private static final int FOOTER_SIZE = 8; + private static final Logger LOGGER = LogUtils.getLogger(); + private static final List SUPPORTED_VERSIONS = Arrays.asList((byte) 1, (byte) 2); + + private final byte[][] buffer = new byte[1024][]; + private final int[] bufferUncompressedSize = new int[1024]; + + private final int[] chunkTimestamps = new int[1024]; + private final ChunkStatus[] statuses = new ChunkStatus[1024]; + + private final LZ4Compressor compressor; + private final LZ4FastDecompressor decompressor; + + public final ReentrantLock fileLock = new ReentrantLock(true); + private final int compressionLevel; + + public boolean closed = false; + public Path path; + private volatile long lastFlushed; + + + public LinearRegionFile(Path file, int compression) throws IOException { + this.path = file; + this.compressionLevel = compression; + this.compressor = LZ4Factory.fastestInstance().fastCompressor(); + this.decompressor = LZ4Factory.fastestInstance().fastDecompressor(); + + File regionFile = new File(this.path.toString()); + + Arrays.fill(this.bufferUncompressedSize, 0); + + if (!regionFile.canRead()) return; + + try (FileInputStream fileStream = new FileInputStream(regionFile); + DataInputStream rawDataStream = new DataInputStream(fileStream)) { + + long superBlock = rawDataStream.readLong(); + if (superBlock != SUPERBLOCK) + throw new RuntimeException("Invalid superblock: " + superBlock + " in " + file); + + byte version = rawDataStream.readByte(); + if (!SUPPORTED_VERSIONS.contains(version)) + throw new RuntimeException("Invalid version: " + version + " in " + file); + + // Skip newestTimestamp (Long) + Compression level (Byte) + Chunk count (Short): Unused. + rawDataStream.skipBytes(11); + + int dataCount = rawDataStream.readInt(); + long fileLength = file.toFile().length(); + if (fileLength != HEADER_SIZE + dataCount + FOOTER_SIZE) + throw new IOException("Invalid file length: " + this.path + " " + fileLength + " " + (HEADER_SIZE + dataCount + FOOTER_SIZE)); + + rawDataStream.skipBytes(8); // Skip data hash (Long): Unused. + + byte[] rawCompressed = new byte[dataCount]; + rawDataStream.readFully(rawCompressed, 0, dataCount); + + superBlock = rawDataStream.readLong(); + if (superBlock != SUPERBLOCK) + throw new IOException("Footer superblock invalid " + this.path); + + try (DataInputStream dataStream = new DataInputStream(new ZstdInputStream(new ByteArrayInputStream(rawCompressed)))) { + + int[] starts = new int[1024]; + for (int i = 0; i < 1024; i++) { + starts[i] = dataStream.readInt(); + dataStream.skipBytes(4); // Skip timestamps (Int): Unused. + } + + for (int i = 0; i < 1024; i++) { + if (starts[i] > 0) { + int size = starts[i]; + byte[] b = new byte[size]; + dataStream.readFully(b, 0, size); + + int maxCompressedLength = this.compressor.maxCompressedLength(size); + byte[] compressed = new byte[maxCompressedLength]; + int compressedLength = this.compressor.compress(b, 0, size, compressed, 0, maxCompressedLength); + b = new byte[compressedLength]; + System.arraycopy(compressed, 0, b, 0, compressedLength); + + this.buffer[i] = b; + this.bufferUncompressedSize[i] = size; + } + } + } + } + } + + public Path getRegionFile() { + return this.path; + } + + public ReentrantLock getFileLock() { + return this.fileLock; + } + + public void flush() throws IOException { + flushWrapper(); // sync + } + + public void flushWrapper() { + try { + save(); + } catch (IOException e) { + LOGGER.error("Failed to flush region file " + path.toAbsolutePath(), e); + } + } + + public boolean doesChunkExist(ChunkPos pos) throws Exception { + throw new Exception("doesChunkExist is a stub"); + } + + private synchronized void save() throws IOException { + long timestamp = getTimestamp(); + short chunkCount = 0; + + File tempFile = new File(path.toString() + ".tmp"); + + try (FileOutputStream fileStream = new FileOutputStream(tempFile); + ByteArrayOutputStream zstdByteArray = new ByteArrayOutputStream(); + ZstdOutputStream zstdStream = new ZstdOutputStream(zstdByteArray, this.compressionLevel); + DataOutputStream zstdDataStream = new DataOutputStream(zstdStream); + DataOutputStream dataStream = new DataOutputStream(fileStream)) { + + dataStream.writeLong(SUPERBLOCK); + dataStream.writeByte(VERSION); + dataStream.writeLong(timestamp); + dataStream.writeByte(this.compressionLevel); + + ArrayList byteBuffers = new ArrayList<>(); + for (int i = 0; i < 1024; i++) { + if (this.bufferUncompressedSize[i] != 0) { + chunkCount += 1; + byte[] content = new byte[bufferUncompressedSize[i]]; + this.decompressor.decompress(buffer[i], 0, content, 0, bufferUncompressedSize[i]); + + byteBuffers.add(content); + } else byteBuffers.add(null); + } + for (int i = 0; i < 1024; i++) { + zstdDataStream.writeInt(this.bufferUncompressedSize[i]); // Write uncompressed size + zstdDataStream.writeInt(this.chunkTimestamps[i]); // Write timestamp + } + for (int i = 0; i < 1024; i++) { + if (byteBuffers.get(i) != null) + zstdDataStream.write(byteBuffers.get(i), 0, byteBuffers.get(i).length); + } + zstdDataStream.close(); + + dataStream.writeShort(chunkCount); + + byte[] compressed = zstdByteArray.toByteArray(); + + dataStream.writeInt(compressed.length); + dataStream.writeLong(0); + + dataStream.write(compressed, 0, compressed.length); + dataStream.writeLong(SUPERBLOCK); + + dataStream.flush(); + fileStream.getFD().sync(); + fileStream.getChannel().force(true); // Ensure atomicity on Btrfs + } + Files.move(tempFile.toPath(), this.path, StandardCopyOption.REPLACE_EXISTING); + this.lastFlushed = System.nanoTime(); + } + + + public void setStatus(int x, int z, ChunkStatus status) { + this.statuses[getChunkIndex(x, z)] = status; + } + + public synchronized void write(ChunkPos pos, ByteBuffer buffer) { + try { + byte[] b = toByteArray(new ByteArrayInputStream(buffer.array())); + int uncompressedSize = b.length; + + int maxCompressedLength = this.compressor.maxCompressedLength(b.length); + byte[] compressed = new byte[maxCompressedLength]; + int compressedLength = this.compressor.compress(b, 0, b.length, compressed, 0, maxCompressedLength); + b = new byte[compressedLength]; + System.arraycopy(compressed, 0, b, 0, compressedLength); + + int index = getChunkIndex(pos.x, pos.z); + this.buffer[index] = b; + this.chunkTimestamps[index] = getTimestamp(); + this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] = uncompressedSize; + } catch (IOException e) { + LOGGER.error("Chunk write IOException " + e + " " + this.path); + } + + if ((System.nanoTime() - this.lastFlushed) > TimeUnit.NANOSECONDS.toSeconds(me.earthme.luminol.config.modules.misc.RegionFormatConfig.linearFlushFrequency)){ + this.flushWrapper(); + } + } + + public DataOutputStream getChunkDataOutputStream(ChunkPos pos) { + return new DataOutputStream(new BufferedOutputStream(new ChunkBuffer(pos))); + } + + private class ChunkBuffer extends ByteArrayOutputStream { + private final ChunkPos pos; + + public ChunkBuffer(ChunkPos chunkcoordintpair) { + super(); + this.pos = chunkcoordintpair; + } + + public void close() throws IOException { + ByteBuffer bytebuffer = ByteBuffer.wrap(this.buf, 0, this.count); + LinearRegionFile.this.write(this.pos, bytebuffer); + } + } + + private byte[] toByteArray(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] tempBuffer = new byte[4096]; + + int length; + while ((length = in.read(tempBuffer)) >= 0) { + out.write(tempBuffer, 0, length); + } + + return out.toByteArray(); + } + + @Nullable + public synchronized DataInputStream getChunkDataInputStream(ChunkPos pos) { + if(this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] != 0) { + byte[] content = new byte[bufferUncompressedSize[getChunkIndex(pos.x, pos.z)]]; + this.decompressor.decompress(this.buffer[getChunkIndex(pos.x, pos.z)], 0, content, 0, bufferUncompressedSize[getChunkIndex(pos.x, pos.z)]); + return new DataInputStream(new ByteArrayInputStream(content)); + } + return null; + } + + public ChunkStatus getStatusIfCached(int x, int z) { + return this.statuses[getChunkIndex(x, z)]; + } + + public void clear(ChunkPos pos) { + int i = getChunkIndex(pos.x, pos.z); + this.buffer[i] = null; + this.bufferUncompressedSize[i] = 0; + this.chunkTimestamps[i] = getTimestamp(); + + if ((System.nanoTime() - this.lastFlushed) > TimeUnit.NANOSECONDS.toSeconds(me.earthme.luminol.config.modules.misc.RegionFormatConfig.linearFlushFrequency)){ + this.flushWrapper(); + } + } + + public boolean hasChunk(ChunkPos pos) { + return this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] > 0; + } + + public void close() throws IOException { + if (closed) return; + closed = true; + flush(); // sync + } + + private static int getChunkIndex(int x, int z) { + return (x & 31) + ((z & 31) << 5); + } + + private static int getTimestamp() { + return (int) (System.currentTimeMillis() / 1000L); + } + + public boolean recalculateHeader() { + return false; + } + + public void setOversized(int x, int z, boolean something) {} + + public CompoundTag getOversizedData(int x, int z) throws IOException { + throw new IOException("getOversizedData is a stub " + this.path); + } + + public boolean isOversized(int x, int z) { + return false; + } +} diff --git a/src/main/java/org/purpurmc/purpur/region/RegionFileFormat.java b/src/main/java/org/purpurmc/purpur/region/RegionFileFormat.java new file mode 100644 index 0000000000000000000000000000000000000000..ebafa01cd90f43ea8c2e183f33f200958543cc1f --- /dev/null +++ b/src/main/java/org/purpurmc/purpur/region/RegionFileFormat.java @@ -0,0 +1,55 @@ +package org.purpurmc.purpur.region; + + +import org.jetbrains.annotations.Contract; +import org.jetbrains.annotations.NotNull; + +public enum RegionFileFormat { + LINEAR(".linear"), + MCA(".mca"), + UNKNOWN(null); + + private final String extensionName; + + RegionFileFormat(String extensionName) { + this.extensionName = extensionName; + } + + public String getExtensionName() { + return this.extensionName; + } + + @Contract(pure = true) + public static RegionFileFormat fromName(@NotNull String name){ + switch (name){ + default -> { + return UNKNOWN; + } + + case "MCA" -> { + return MCA; + } + + case "LINEAR" -> { + return LINEAR; + } + } + } + + @Contract(pure = true) + public static RegionFileFormat fromExtension(@NotNull String name){ + switch (name){ + default -> { + return UNKNOWN; + } + + case ".mca" -> { + return MCA; + } + + case ".linear" -> { + return LINEAR; + } + } + } +}