From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: NONPLAYT <76615486+NONPLAYT@users.noreply.github.com> Date: Thu, 28 Dec 2023 02:39:42 +0300 Subject: [PATCH] Implement Linear region format diff --git a/build.gradle.kts b/build.gradle.kts index 4c4a0b04a994bcbf0ddc183e54ad5f802a81a4cc..c8307d1ef3a6490649c7ecefe28db9865c3567a2 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -36,6 +36,10 @@ dependencies { alsoShade(log4jPlugins.output) implementation("io.netty:netty-codec-haproxy:4.1.97.Final") // Paper - Add support for proxy protocol // Paper end + // DivineMC start - Implement Linear region + implementation("com.github.luben:zstd-jni:1.5.5-11") + implementation("org.lz4:lz4-java:1.8.0") + // DivineMC end implementation("org.apache.logging.log4j:log4j-iostreams:2.19.0") // Paper - remove exclusion implementation("org.ow2.asm:asm:9.5") implementation("org.ow2.asm:asm-commons:9.5") // Paper - ASM event executor generation diff --git a/src/main/java/com/destroystokyo/paper/io/PaperFileIOThread.java b/src/main/java/com/destroystokyo/paper/io/PaperFileIOThread.java index f2c27e0ac65be4b75c1d86ef6fd45fdb538d96ac..84ef82da3a18341c1f8bc463360e8ba031299da4 100644 --- a/src/main/java/com/destroystokyo/paper/io/PaperFileIOThread.java +++ b/src/main/java/com/destroystokyo/paper/io/PaperFileIOThread.java @@ -1,10 +1,9 @@ package com.destroystokyo.paper.io; import com.mojang.logging.LogUtils; +import gq.bxteam.divinemc.region.AbstractRegionFile; import net.minecraft.nbt.CompoundTag; import net.minecraft.server.level.ServerLevel; -import net.minecraft.world.level.ChunkPos; -import net.minecraft.world.level.chunk.storage.RegionFile; import org.slf4j.Logger; import java.io.IOException; @@ -314,8 +313,10 @@ public final class PaperFileIOThread extends QueueExecutorThread { public abstract void writeData(final int x, final int z, final CompoundTag compound) throws IOException; public abstract CompoundTag readData(final int x, final int z) throws IOException; - public abstract T computeForRegionFile(final int chunkX, final int chunkZ, final Function function); - public abstract T computeForRegionFileIfLoaded(final int chunkX, final int chunkZ, final Function function); + // DivineMC start - Implement Linear region + public abstract T computeForRegionFile(final int chunkX, final int chunkZ, final Function function); + public abstract T computeForRegionFileIfLoaded(final int chunkX, final int chunkZ, final Function function); + // DivineMC end public static final class InProgressWrite { public long writeCounter; diff --git a/src/main/java/gq/bxteam/divinemc/configuration/DivineConfig.java b/src/main/java/gq/bxteam/divinemc/configuration/DivineConfig.java index 11dc1af9f8d8ce8c0a855d14a35077f5482ef0e8..def6f834d32b97e1065b17657e73d020a873bd14 100644 --- a/src/main/java/gq/bxteam/divinemc/configuration/DivineConfig.java +++ b/src/main/java/gq/bxteam/divinemc/configuration/DivineConfig.java @@ -157,4 +157,15 @@ public class DivineConfig { private static void doNotProcessChatCommands() { doNotProcessChatCommands = getBoolean("settings.do-not-process-chat-commands", doNotProcessChatCommands); } + + public static int linearFlushFrequency = 10; + public static int linearFlushThreads = 1; + private static void regionFormatSettings() { + linearFlushFrequency = getInt("region-format.linear.flush-frequency", linearFlushFrequency); + linearFlushThreads = getInt("region-format.linear.flush-max-threads", linearFlushThreads); + if (linearFlushThreads < 0) + linearFlushThreads = Math.max(Runtime.getRuntime().availableProcessors() + linearFlushThreads, 1); + else + linearFlushThreads = Math.max(linearFlushThreads, 1); + } } diff --git a/src/main/java/gq/bxteam/divinemc/configuration/DivineWorldConfig.java b/src/main/java/gq/bxteam/divinemc/configuration/DivineWorldConfig.java index ca344da743a7503795bdaeff0a1b14e0721f5092..7deda3f07cbe5c520ae44575d79f4e857f0b44a1 100644 --- a/src/main/java/gq/bxteam/divinemc/configuration/DivineWorldConfig.java +++ b/src/main/java/gq/bxteam/divinemc/configuration/DivineWorldConfig.java @@ -1,5 +1,6 @@ package gq.bxteam.divinemc.configuration; +import gq.bxteam.divinemc.region.RegionFileFormat; import org.apache.commons.lang.BooleanUtils; import org.bukkit.World; import org.bukkit.configuration.ConfigurationSection; @@ -7,6 +8,7 @@ import org.bukkit.configuration.ConfigurationSection; import java.util.List; import java.util.Map; import java.util.function.Predicate; +import java.util.logging.Level; import static gq.bxteam.divinemc.configuration.DivineConfig.log; @@ -89,4 +91,23 @@ public class DivineWorldConfig { private void despawnShulkerBulletsOnOwnerDeath() { despawnShulkerBulletsOnOwnerDeath = getBoolean("gameplay-mechanics.mob.shulker.despawn-bullets-on-player-death", despawnShulkerBulletsOnOwnerDeath); } + + public RegionFileFormat regionFormatName = RegionFileFormat.ANVIL; + public boolean linearCrashOnBrokenSymlink = true; + public int regionFormatLinearCompressionLevel = 1; + private void regionFormatSettings() { + regionFormatName = RegionFileFormat.fromString(getString("region-format.format", regionFormatName.name())); + if (regionFormatName.equals(RegionFileFormat.INVALID)) { + DivineConfig.log(Level.SEVERE, "Unknown region format in purpur.yml: " + regionFormatName); + DivineConfig.log(Level.SEVERE, "Falling back to ANVIL region file format."); + regionFormatName = RegionFileFormat.ANVIL; + } + regionFormatLinearCompressionLevel = getInt("region-format.linear.compression-level", regionFormatLinearCompressionLevel); + if (regionFormatLinearCompressionLevel > 23 || regionFormatLinearCompressionLevel < 1) { + DivineConfig.log(Level.SEVERE, "Linear region compression level should be between 1 and 22 in purpur.yml: " + regionFormatLinearCompressionLevel); + DivineConfig.log(Level.SEVERE, "Falling back to compression level 1."); + regionFormatLinearCompressionLevel = 1; + } + linearCrashOnBrokenSymlink = getBoolean("region-format.linear.crash-on-broken-symlink", linearCrashOnBrokenSymlink); + } } \ No newline at end of file diff --git a/src/main/java/gq/bxteam/divinemc/region/AbstractRegionFile.java b/src/main/java/gq/bxteam/divinemc/region/AbstractRegionFile.java new file mode 100644 index 0000000000000000000000000000000000000000..690c9fe4a3983a09463a1ad6da46173d5d7647eb --- /dev/null +++ b/src/main/java/gq/bxteam/divinemc/region/AbstractRegionFile.java @@ -0,0 +1,31 @@ +package gq.bxteam.divinemc.region; + +import net.minecraft.nbt.CompoundTag; +import net.minecraft.world.level.ChunkPos; +import net.minecraft.world.level.chunk.ChunkStatus; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.file.Path; +import java.util.concurrent.locks.ReentrantLock; + +public interface AbstractRegionFile { + void flush() throws IOException; + void clear(ChunkPos pos) throws IOException; + void close() throws IOException; + void setStatus(int x, int z, ChunkStatus status); + void setOversized(int x, int z, boolean b) throws IOException; + + boolean hasChunk(ChunkPos pos); + boolean doesChunkExist(ChunkPos pos) throws Exception; + boolean isOversized(int x, int z); + boolean recalculateHeader() throws IOException; + + DataOutputStream getChunkDataOutputStream(ChunkPos pos) throws IOException; + DataInputStream getChunkDataInputStream(ChunkPos pos) throws IOException; + CompoundTag getOversizedData(int x, int z) throws IOException; + ChunkStatus getStatusIfCached(int x, int z); + ReentrantLock getFileLock(); + Path getRegionFile(); +} \ No newline at end of file diff --git a/src/main/java/gq/bxteam/divinemc/region/AbstractRegionFileFactory.java b/src/main/java/gq/bxteam/divinemc/region/AbstractRegionFileFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..9380b325bb21a65996d27d3353b02c1ec22915b2 --- /dev/null +++ b/src/main/java/gq/bxteam/divinemc/region/AbstractRegionFileFactory.java @@ -0,0 +1,29 @@ +package gq.bxteam.divinemc.region; + +import net.minecraft.world.level.chunk.storage.RegionFile; +import net.minecraft.world.level.chunk.storage.RegionFileVersion; + +import java.io.IOException; +import java.nio.file.Path; + +public class AbstractRegionFileFactory { + public static AbstractRegionFile getAbstractRegionFile(int linearCompression, Path file, Path directory, boolean dsync) throws IOException { + return getAbstractRegionFile(linearCompression, file, directory, RegionFileVersion.VERSION_DEFLATE, dsync); + } + + public static AbstractRegionFile getAbstractRegionFile(int linearCompression, Path file, Path directory, boolean dsync, boolean canRecalcHeader) throws IOException { + return getAbstractRegionFile(linearCompression, file, directory, RegionFileVersion.VERSION_DEFLATE, dsync, canRecalcHeader); + } + + public static AbstractRegionFile getAbstractRegionFile(int linearCompression, Path file, Path directory, RegionFileVersion outputChunkStreamVersion, boolean dsync) throws IOException { + return getAbstractRegionFile(linearCompression, file, directory, outputChunkStreamVersion, dsync, false); + } + + public static AbstractRegionFile getAbstractRegionFile(int linearCompression, Path file, Path directory, RegionFileVersion outputChunkStreamVersion, boolean dsync, boolean canRecalcHeader) throws IOException { + if (file.toString().endsWith(".linear")) { + return new LinearRegionFile(file, linearCompression); + } else { + return new RegionFile(file, directory, outputChunkStreamVersion, dsync, canRecalcHeader); + } + } +} diff --git a/src/main/java/gq/bxteam/divinemc/region/LinearRegionFile.java b/src/main/java/gq/bxteam/divinemc/region/LinearRegionFile.java new file mode 100644 index 0000000000000000000000000000000000000000..2c82bb08127f729a43e5c1a7a4c8969b4e7a3e70 --- /dev/null +++ b/src/main/java/gq/bxteam/divinemc/region/LinearRegionFile.java @@ -0,0 +1,315 @@ +package gq.bxteam.divinemc.region; + +import com.github.luben.zstd.ZstdInputStream; +import com.github.luben.zstd.ZstdOutputStream; +import com.mojang.logging.LogUtils; +import net.jpountz.lz4.LZ4Compressor; +import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4FastDecompressor; +import net.minecraft.nbt.CompoundTag; +import net.minecraft.world.level.ChunkPos; +import net.minecraft.world.level.chunk.ChunkStatus; +import org.slf4j.Logger; + +import javax.annotation.Nullable; +import java.io.*; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantLock; + +public class LinearRegionFile implements AbstractRegionFile, AutoCloseable { + private static final long SUPERBLOCK = -4323716122432332390L; + private static final byte VERSION = 2; + private static final int HEADER_SIZE = 32; + private static final int FOOTER_SIZE = 8; + private static final Logger LOGGER = LogUtils.getLogger(); + private static final List SUPPORTED_VERSIONS = Arrays.asList((byte) 1, (byte) 2); + private static final LinearRegionFileFlusher linearRegionFileFlusher = new LinearRegionFileFlusher(); + + private final byte[][] buffer = new byte[1024][]; + private final int[] bufferUncompressedSize = new int[1024]; + + private final int[] chunkTimestamps = new int[1024]; + private final ChunkStatus[] statuses = new ChunkStatus[1024]; + + private final LZ4Compressor compressor; + private final LZ4FastDecompressor decompressor; + + public final ReentrantLock fileLock = new ReentrantLock(true); + private final int compressionLevel; + + private AtomicBoolean markedToSave = new AtomicBoolean(false); + public boolean closed = false; + public Path path; + + + public LinearRegionFile(Path file, int compression) throws IOException { + this.path = file; + this.compressionLevel = compression; + this.compressor = LZ4Factory.fastestInstance().fastCompressor(); + this.decompressor = LZ4Factory.fastestInstance().fastDecompressor(); + + File regionFile = new File(this.path.toString()); + + Arrays.fill(this.bufferUncompressedSize, 0); + + if (!regionFile.canRead()) return; + + try (FileInputStream fileStream = new FileInputStream(regionFile); + DataInputStream rawDataStream = new DataInputStream(fileStream)) { + + long superBlock = rawDataStream.readLong(); + if (superBlock != SUPERBLOCK) + throw new RuntimeException("Invalid superblock: " + superBlock + " in " + file); + + byte version = rawDataStream.readByte(); + if (!SUPPORTED_VERSIONS.contains(version)) + throw new RuntimeException("Invalid version: " + version + " in " + file); + + // Skip newestTimestamp (Long) + Compression level (Byte) + Chunk count (Short): Unused. + rawDataStream.skipBytes(11); + + int dataCount = rawDataStream.readInt(); + long fileLength = file.toFile().length(); + if (fileLength != HEADER_SIZE + dataCount + FOOTER_SIZE) + throw new IOException("Invalid file length: " + this.path + " " + fileLength + " " + (HEADER_SIZE + dataCount + FOOTER_SIZE)); + + rawDataStream.skipBytes(8); // Skip data hash (Long): Unused. + + byte[] rawCompressed = new byte[dataCount]; + rawDataStream.readFully(rawCompressed, 0, dataCount); + + superBlock = rawDataStream.readLong(); + if (superBlock != SUPERBLOCK) + throw new IOException("Footer superblock invalid " + this.path); + + try (DataInputStream dataStream = new DataInputStream(new ZstdInputStream(new ByteArrayInputStream(rawCompressed)))) { + + int[] starts = new int[1024]; + for (int i = 0; i < 1024; i++) { + starts[i] = dataStream.readInt(); + dataStream.skipBytes(4); // Skip timestamps (Int): Unused. + } + + for (int i = 0; i < 1024; i++) { + if (starts[i] > 0) { + int size = starts[i]; + byte[] b = new byte[size]; + dataStream.readFully(b, 0, size); + + int maxCompressedLength = this.compressor.maxCompressedLength(size); + byte[] compressed = new byte[maxCompressedLength]; + int compressedLength = this.compressor.compress(b, 0, size, compressed, 0, maxCompressedLength); + b = new byte[compressedLength]; + System.arraycopy(compressed, 0, b, 0, compressedLength); + + this.buffer[i] = b; + this.bufferUncompressedSize[i] = size; + } + } + } + } + } + + public Path getRegionFile() { + return this.path; + } + + public ReentrantLock getFileLock() { + return this.fileLock; + } + + public void flush() throws IOException { + if (isMarkedToSave()) flushWrapper(); // sync + } + + private void markToSave() { + linearRegionFileFlusher.scheduleSave(this); + markedToSave.set(true); + } + + public boolean isMarkedToSave() { + return markedToSave.getAndSet(false); + } + + public void flushWrapper() { + try { + save(); + } catch (IOException e) { + LOGGER.error("Failed to flush region file " + path.toAbsolutePath(), e); + } + } + + public boolean doesChunkExist(ChunkPos pos) throws Exception { + throw new Exception("doesChunkExist is a stub"); + } + + private synchronized void save() throws IOException { + long timestamp = getTimestamp(); + short chunkCount = 0; + + File tempFile = new File(path.toString() + ".tmp"); + + try (FileOutputStream fileStream = new FileOutputStream(tempFile); + ByteArrayOutputStream zstdByteArray = new ByteArrayOutputStream(); + ZstdOutputStream zstdStream = new ZstdOutputStream(zstdByteArray, this.compressionLevel); + DataOutputStream zstdDataStream = new DataOutputStream(zstdStream); + DataOutputStream dataStream = new DataOutputStream(fileStream)) { + + dataStream.writeLong(SUPERBLOCK); + dataStream.writeByte(VERSION); + dataStream.writeLong(timestamp); + dataStream.writeByte(this.compressionLevel); + + ArrayList byteBuffers = new ArrayList<>(); + for (int i = 0; i < 1024; i++) { + if (this.bufferUncompressedSize[i] != 0) { + chunkCount += 1; + byte[] content = new byte[bufferUncompressedSize[i]]; + this.decompressor.decompress(buffer[i], 0, content, 0, bufferUncompressedSize[i]); + + byteBuffers.add(content); + } else byteBuffers.add(null); + } + for (int i = 0; i < 1024; i++) { + zstdDataStream.writeInt(this.bufferUncompressedSize[i]); // Write uncompressed size + zstdDataStream.writeInt(this.chunkTimestamps[i]); // Write timestamp + } + for (int i = 0; i < 1024; i++) { + if (byteBuffers.get(i) != null) + zstdDataStream.write(byteBuffers.get(i), 0, byteBuffers.get(i).length); + } + zstdDataStream.close(); + + dataStream.writeShort(chunkCount); + + byte[] compressed = zstdByteArray.toByteArray(); + + dataStream.writeInt(compressed.length); + dataStream.writeLong(0); + + dataStream.write(compressed, 0, compressed.length); + dataStream.writeLong(SUPERBLOCK); + + dataStream.flush(); + fileStream.getFD().sync(); + fileStream.getChannel().force(true); // Ensure atomicity on Btrfs + } + Files.move(tempFile.toPath(), this.path, StandardCopyOption.REPLACE_EXISTING); + } + + public void setStatus(int x, int z, ChunkStatus status) { + this.statuses[getChunkIndex(x, z)] = status; + } + + public synchronized void write(ChunkPos pos, ByteBuffer buffer) { + try { + byte[] b = toByteArray(new ByteArrayInputStream(buffer.array())); + int uncompressedSize = b.length; + + int maxCompressedLength = this.compressor.maxCompressedLength(b.length); + byte[] compressed = new byte[maxCompressedLength]; + int compressedLength = this.compressor.compress(b, 0, b.length, compressed, 0, maxCompressedLength); + b = new byte[compressedLength]; + System.arraycopy(compressed, 0, b, 0, compressedLength); + + int index = getChunkIndex(pos.x, pos.z); + this.buffer[index] = b; + this.chunkTimestamps[index] = getTimestamp(); + this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] = uncompressedSize; + } catch (IOException e) { + LOGGER.error("Chunk write IOException " + e + " " + this.path); + } + markToSave(); + } + + public DataOutputStream getChunkDataOutputStream(ChunkPos pos) { + return new DataOutputStream(new BufferedOutputStream(new ChunkBuffer(pos))); + } + + private class ChunkBuffer extends ByteArrayOutputStream { + private final ChunkPos pos; + + public ChunkBuffer(ChunkPos chunkcoordintpair) { + super(); + this.pos = chunkcoordintpair; + } + + public void close() throws IOException { + ByteBuffer bytebuffer = ByteBuffer.wrap(this.buf, 0, this.count); + LinearRegionFile.this.write(this.pos, bytebuffer); + } + } + + private byte[] toByteArray(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] tempBuffer = new byte[4096]; + + int length; + while ((length = in.read(tempBuffer)) >= 0) { + out.write(tempBuffer, 0, length); + } + + return out.toByteArray(); + } + + @Nullable + public synchronized DataInputStream getChunkDataInputStream(ChunkPos pos) { + if(this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] != 0) { + byte[] content = new byte[bufferUncompressedSize[getChunkIndex(pos.x, pos.z)]]; + this.decompressor.decompress(this.buffer[getChunkIndex(pos.x, pos.z)], 0, content, 0, bufferUncompressedSize[getChunkIndex(pos.x, pos.z)]); + return new DataInputStream(new ByteArrayInputStream(content)); + } + return null; + } + + public ChunkStatus getStatusIfCached(int x, int z) { + return this.statuses[getChunkIndex(x, z)]; + } + + public void clear(ChunkPos pos) { + int i = getChunkIndex(pos.x, pos.z); + this.buffer[i] = null; + this.bufferUncompressedSize[i] = 0; + this.chunkTimestamps[i] = getTimestamp(); + markToSave(); + } + + public boolean hasChunk(ChunkPos pos) { + return this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] > 0; + } + + public void close() throws IOException { + if (closed) return; + closed = true; + flush(); // sync + } + + private static int getChunkIndex(int x, int z) { + return (x & 31) + ((z & 31) << 5); + } + + private static int getTimestamp() { + return (int) (System.currentTimeMillis() / 1000L); + } + + public boolean recalculateHeader() { + return false; + } + + public void setOversized(int x, int z, boolean something) {} + + public CompoundTag getOversizedData(int x, int z) throws IOException { + throw new IOException("getOversizedData is a stub " + this.path); + } + + public boolean isOversized(int x, int z) { + return false; + } +} \ No newline at end of file diff --git a/src/main/java/gq/bxteam/divinemc/region/LinearRegionFileFlusher.java b/src/main/java/gq/bxteam/divinemc/region/LinearRegionFileFlusher.java new file mode 100644 index 0000000000000000000000000000000000000000..efae9dc7a037ef8bf45f8d9439ae710b009e8ec8 --- /dev/null +++ b/src/main/java/gq/bxteam/divinemc/region/LinearRegionFileFlusher.java @@ -0,0 +1,46 @@ +package gq.bxteam.divinemc.region; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.Queue; +import java.util.concurrent.*; + +import gq.bxteam.divinemc.configuration.DivineConfig; +import org.bukkit.Bukkit; + +public class LinearRegionFileFlusher { + private final Queue savingQueue = new LinkedBlockingQueue<>(); + private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder() + .setNameFormat("linear-flush-scheduler") + .build() + ); + private final ExecutorService executor = Executors.newFixedThreadPool( + DivineConfig.linearFlushThreads, + new ThreadFactoryBuilder() + .setNameFormat("linear-flusher-%d") + .build() + ); + + public LinearRegionFileFlusher() { + Bukkit.getLogger().info("Using " + DivineConfig.linearFlushThreads + " threads for linear region flushing."); + scheduler.scheduleAtFixedRate(this::pollAndFlush, 0L, DivineConfig.linearFlushFrequency, TimeUnit.SECONDS); + } + + public void scheduleSave(LinearRegionFile regionFile) { + if (savingQueue.contains(regionFile)) return; + savingQueue.add(regionFile); + } + + private void pollAndFlush() { + while (!savingQueue.isEmpty()) { + LinearRegionFile regionFile = savingQueue.poll(); + if (!regionFile.closed && regionFile.isMarkedToSave()) + executor.execute(regionFile::flushWrapper); + } + } + + public void shutdown() { + executor.shutdown(); + scheduler.shutdown(); + } +} \ No newline at end of file diff --git a/src/main/java/gq/bxteam/divinemc/region/RegionFileFormat.java b/src/main/java/gq/bxteam/divinemc/region/RegionFileFormat.java new file mode 100644 index 0000000000000000000000000000000000000000..73468daa328a330e432fc2be3721a5c5cb8c9df8 --- /dev/null +++ b/src/main/java/gq/bxteam/divinemc/region/RegionFileFormat.java @@ -0,0 +1,16 @@ +package gq.bxteam.divinemc.region; + +public enum RegionFileFormat { + ANVIL, + LINEAR, + INVALID; + + public static RegionFileFormat fromString(String format) { + for (RegionFileFormat rff : values()) { + if (rff.name().equalsIgnoreCase(format)) { + return rff; + } + } + return RegionFileFormat.INVALID; + } +} diff --git a/src/main/java/io/papermc/paper/chunk/system/io/RegionFileIOThread.java b/src/main/java/io/papermc/paper/chunk/system/io/RegionFileIOThread.java index 8a11e10b01fa012b2f98b1c193c53251e848f909..2b3c6d11421bb0f5131eeaae362ee44b2dd7d485 100644 --- a/src/main/java/io/papermc/paper/chunk/system/io/RegionFileIOThread.java +++ b/src/main/java/io/papermc/paper/chunk/system/io/RegionFileIOThread.java @@ -7,6 +7,7 @@ import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedQueueExecutorT import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedThreadedTaskQueue; import ca.spottedleaf.concurrentutil.util.ConcurrentUtil; import com.mojang.logging.LogUtils; +import gq.bxteam.divinemc.region.AbstractRegionFile; import io.papermc.paper.util.CoordinateUtils; import io.papermc.paper.util.TickThread; import it.unimi.dsi.fastutil.HashCommon; @@ -811,7 +812,7 @@ public final class RegionFileIOThread extends PrioritisedQueueExecutorThread { final ChunkDataController taskController) { final ChunkPos chunkPos = new ChunkPos(chunkX, chunkZ); if (intendingToBlock) { - return taskController.computeForRegionFile(chunkX, chunkZ, true, (final RegionFile file) -> { + return taskController.computeForRegionFile(chunkX, chunkZ, true, (final AbstractRegionFile file) -> { // DivineMC - Implement Linear region if (file == null) { // null if no regionfile exists return Boolean.FALSE; } @@ -824,7 +825,7 @@ public final class RegionFileIOThread extends PrioritisedQueueExecutorThread { return Boolean.FALSE; } // else: it either exists or is not known, fall back to checking the loaded region file - return taskController.computeForRegionFileIfLoaded(chunkX, chunkZ, (final RegionFile file) -> { + return taskController.computeForRegionFileIfLoaded(chunkX, chunkZ, (final AbstractRegionFile file) -> { // DivineMC - Implement Linear region if (file == null) { // null if not loaded // not sure at this point, let the I/O thread figure it out return Boolean.TRUE; @@ -1126,9 +1127,9 @@ public final class RegionFileIOThread extends PrioritisedQueueExecutorThread { return this.getCache().doesRegionFileNotExistNoIO(new ChunkPos(chunkX, chunkZ)); } - public T computeForRegionFile(final int chunkX, final int chunkZ, final boolean existingOnly, final Function function) { + public T computeForRegionFile(final int chunkX, final int chunkZ, final boolean existingOnly, final Function function) { // DivineMC - Implement Linear region final RegionFileStorage cache = this.getCache(); - final RegionFile regionFile; + final AbstractRegionFile regionFile; // DivineMC - Implement Linear region synchronized (cache) { try { regionFile = cache.getRegionFile(new ChunkPos(chunkX, chunkZ), existingOnly, true); @@ -1141,19 +1142,19 @@ public final class RegionFileIOThread extends PrioritisedQueueExecutorThread { return function.apply(regionFile); } finally { if (regionFile != null) { - regionFile.fileLock.unlock(); + regionFile.getFileLock().unlock(); // DivineMC - Implement Linear region } } } - public T computeForRegionFileIfLoaded(final int chunkX, final int chunkZ, final Function function) { + public T computeForRegionFileIfLoaded(final int chunkX, final int chunkZ, final Function function) { final RegionFileStorage cache = this.getCache(); - final RegionFile regionFile; + final AbstractRegionFile regionFile; synchronized (cache) { regionFile = cache.getRegionFileIfLoaded(new ChunkPos(chunkX, chunkZ)); if (regionFile != null) { - regionFile.fileLock.lock(); + regionFile.getFileLock().lock(); // DivineMC - Implement Linear region } } @@ -1161,7 +1162,7 @@ public final class RegionFileIOThread extends PrioritisedQueueExecutorThread { return function.apply(regionFile); } finally { if (regionFile != null) { - regionFile.fileLock.unlock(); + regionFile.getFileLock().unlock(); // DivineMC - Implement Linear region } } } diff --git a/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java index 513833c2ea23df5b079d157bc5cb89d5c9754c0b..b48b7d538132ad3d51d373292e36a8a10a9c0361 100644 --- a/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java +++ b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java @@ -2,22 +2,23 @@ package io.papermc.paper.world; import com.mojang.datafixers.DataFixer; import com.mojang.serialization.Codec; +import gq.bxteam.divinemc.region.RegionFileFormat; import net.minecraft.SharedConstants; import net.minecraft.nbt.CompoundTag; import net.minecraft.resources.ResourceKey; import net.minecraft.util.worldupdate.WorldUpgrader; import net.minecraft.world.level.ChunkPos; -import net.minecraft.world.level.Level; import net.minecraft.world.level.chunk.ChunkGenerator; import net.minecraft.world.level.chunk.storage.ChunkStorage; import net.minecraft.world.level.chunk.storage.RegionFileStorage; -import net.minecraft.world.level.dimension.DimensionType; import net.minecraft.world.level.dimension.LevelStem; -import net.minecraft.world.level.levelgen.WorldGenSettings; import net.minecraft.world.level.storage.DimensionDataStorage; import net.minecraft.world.level.storage.LevelStorageSource; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.bukkit.Bukkit; +import org.bukkit.craftbukkit.CraftWorld; + import java.io.File; import java.io.IOException; import java.text.DecimalFormat; @@ -84,8 +85,15 @@ public class ThreadedWorldUpgrader { LOGGER.info("Found " + regionFiles.length + " regionfiles to convert"); LOGGER.info("Starting conversion now for world " + this.worldName); + // DivineMC start - Implement Linear region + RegionFileFormat formatName = ((CraftWorld) Bukkit.getWorld(worldName)).getHandle().divinemcConfig.regionFormatName; + int linearCompression = ((org.bukkit.craftbukkit.CraftWorld) org.bukkit.Bukkit.getWorld(worldName)).getHandle().divinemcConfig.regionFormatLinearCompressionLevel; + boolean linearCrashOnBrokenSymlink = ((org.bukkit.craftbukkit.CraftWorld) org.bukkit.Bukkit.getWorld(worldName)).getHandle().divinemcConfig.linearCrashOnBrokenSymlink; + LOGGER.info("Using format " + formatName + " (" + linearCompression + ")"); + // DivineMC end + final WorldInfo info = new WorldInfo(() -> worldPersistentData, - new ChunkStorage(regionFolder.toPath(), this.dataFixer, false), this.removeCaches, this.dimensionType, this.generatorKey); + new ChunkStorage(formatName, linearCompression, linearCrashOnBrokenSymlink, regionFolder.toPath(), this.dataFixer, false), this.removeCaches, this.dimensionType, this.generatorKey); // DivineMC - Implement Linear region long expectedChunks = (long)regionFiles.length * (32L * 32L); diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java index f69976dcba060027c67c2e1b49fa28d3f28f66f0..e10ef28420c755258f7fbe7935f8c8ab0fdcca50 100644 --- a/src/main/java/net/minecraft/server/MinecraftServer.java +++ b/src/main/java/net/minecraft/server/MinecraftServer.java @@ -887,7 +887,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop mainThreadExecutor, LightChunkGetter chunkProvider, ChunkGenerator chunkGenerator, ChunkProgressListener worldGenerationProgressListener, ChunkStatusUpdateListener chunkStatusChangeListener, Supplier persistentStateManagerFactory, int viewDistance, boolean dsync) { - super(session.getDimensionPath(world.dimension()).resolve("region"), dataFixer, dsync); + super(world.getLevel().divinemcConfig.regionFormatName, world.getLevel().divinemcConfig.regionFormatLinearCompressionLevel, world.getLevel().divinemcConfig.linearCrashOnBrokenSymlink, session.getDimensionPath(world.dimension()).resolve("region"), dataFixer, dsync); // DivineMC - Implement Linear region // Paper - rewrite chunk system this.tickingGenerated = new AtomicInteger(); this.playerMap = new PlayerMap(); @@ -292,7 +274,7 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider this.lightEngine = new ThreadedLevelLightEngine(chunkProvider, this, this.level.dimensionType().hasSkyLight(), null, null); // Paper - rewrite chunk system this.distanceManager = new ChunkMap.ChunkDistanceManager(executor, mainThreadExecutor); this.overworldDataStorage = persistentStateManagerFactory; - this.poiManager = new PoiManager(path.resolve("poi"), dataFixer, dsync, iregistrycustom, world); + this.poiManager = new PoiManager(this.level.divinemcConfig.regionFormatName, this.level.divinemcConfig.regionFormatLinearCompressionLevel, this.level.divinemcConfig.linearCrashOnBrokenSymlink, path.resolve("poi"), dataFixer, dsync, iregistrycustom, world); // DivineMC - Implement Linear region this.setServerViewDistance(viewDistance); // Paper start this.dataRegionManager = new io.papermc.paper.chunk.SingleThreadChunkRegionManager(this.level, 2, (1.0 / 3.0), 1, 6, "Data", DataRegionData::new, DataRegionSectionData::new); @@ -886,13 +868,13 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider // Paper start - chunk status cache "api" public ChunkStatus getChunkStatusOnDiskIfCached(ChunkPos chunkPos) { - net.minecraft.world.level.chunk.storage.RegionFile regionFile = regionFileCache.getRegionFileIfLoaded(chunkPos); + AbstractRegionFile regionFile = regionFileCache.getRegionFileIfLoaded(chunkPos); // DivineMC - Implement Linear region return regionFile == null ? null : regionFile.getStatusIfCached(chunkPos.x, chunkPos.z); } public ChunkStatus getChunkStatusOnDisk(ChunkPos chunkPos) throws IOException { - net.minecraft.world.level.chunk.storage.RegionFile regionFile = regionFileCache.getRegionFile(chunkPos, true); + AbstractRegionFile regionFile = regionFileCache.getRegionFile(chunkPos, true); // DivineMC - Implement Linear region if (regionFile == null || !regionFileCache.chunkExists(chunkPos)) { return null; @@ -910,7 +892,7 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider } public void updateChunkStatusOnDisk(ChunkPos chunkPos, @Nullable CompoundTag compound) throws IOException { - net.minecraft.world.level.chunk.storage.RegionFile regionFile = regionFileCache.getRegionFile(chunkPos, false); + AbstractRegionFile regionFile = regionFileCache.getRegionFile(chunkPos, false); // DivineMC - Implement Linear region regionFile.setStatus(chunkPos.x, chunkPos.z, ChunkSerializer.getStatus(compound)); } diff --git a/src/main/java/net/minecraft/server/level/ServerLevel.java b/src/main/java/net/minecraft/server/level/ServerLevel.java index 90000d74cfb009e2cbd7336ae11077d67ef67f7b..34ecc8e5bf5f8443c6046d0b61f60dd6cb521872 100644 --- a/src/main/java/net/minecraft/server/level/ServerLevel.java +++ b/src/main/java/net/minecraft/server/level/ServerLevel.java @@ -6,6 +6,7 @@ import com.google.common.collect.Lists; import com.mojang.datafixers.DataFixer; import com.mojang.datafixers.util.Pair; import com.mojang.logging.LogUtils; +import gq.bxteam.divinemc.region.RegionFileFormat; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; import it.unimi.dsi.fastutil.longs.LongSet; @@ -50,7 +51,6 @@ import net.minecraft.core.particles.ParticleOptions; import net.minecraft.core.registries.BuiltInRegistries; import net.minecraft.core.registries.Registries; import net.minecraft.network.chat.Component; -import net.minecraft.network.chat.MutableComponent; import net.minecraft.network.protocol.Packet; import net.minecraft.network.protocol.game.ClientboundBlockDestructionPacket; import net.minecraft.network.protocol.game.ClientboundBlockEventPacket; @@ -78,7 +78,6 @@ import net.minecraft.util.Mth; import net.minecraft.util.ProgressListener; import net.minecraft.util.RandomSource; import net.minecraft.util.Unit; -import net.minecraft.util.profiling.ProfilerFiller; import net.minecraft.util.valueproviders.IntProvider; import net.minecraft.util.valueproviders.UniformInt; import net.minecraft.world.DifficultyInstance; @@ -134,12 +133,10 @@ import net.minecraft.world.level.chunk.storage.EntityStorage; import net.minecraft.world.level.dimension.BuiltinDimensionTypes; import net.minecraft.world.level.dimension.LevelStem; import net.minecraft.world.level.dimension.end.EndDragonFight; -import net.minecraft.world.level.entity.EntityPersistentStorage; import net.minecraft.world.level.entity.EntityTickList; import net.minecraft.world.level.entity.EntityTypeTest; import net.minecraft.world.level.entity.LevelCallback; import net.minecraft.world.level.entity.LevelEntityGetter; -import net.minecraft.world.level.entity.PersistentEntitySectionManager; import net.minecraft.world.level.gameevent.DynamicGameEventListener; import net.minecraft.world.level.gameevent.GameEvent; import net.minecraft.world.level.gameevent.GameEventDispatcher; @@ -170,12 +167,10 @@ import org.bukkit.Location; import org.bukkit.WeatherType; import org.bukkit.craftbukkit.event.CraftEventFactory; import org.bukkit.craftbukkit.generator.CustomWorldChunkManager; -import org.bukkit.craftbukkit.util.CraftNamespacedKey; import org.bukkit.craftbukkit.util.WorldUUID; import org.bukkit.event.entity.CreatureSpawnEvent; import org.bukkit.event.server.MapInitializeEvent; import org.bukkit.event.weather.LightningStrikeEvent; -import org.bukkit.event.world.GenericGameEvent; import org.bukkit.event.world.TimeSkipEvent; // CraftBukkit end @@ -426,9 +421,11 @@ public class ServerLevel extends Level implements WorldGenLevel { private static final class EntityRegionFileStorage extends net.minecraft.world.level.chunk.storage.RegionFileStorage { - public EntityRegionFileStorage(Path directory, boolean dsync) { - super(directory, dsync); + // DivineMC start - Implement Linear region + public EntityRegionFileStorage(RegionFileFormat format, int linearCompression, boolean linearCrashOnBrokenSymlink, Path directory, boolean dsync) { // LinearPurpur + super(format, linearCompression, linearCrashOnBrokenSymlink, directory, dsync); } + // DivineMC end protected void write(ChunkPos pos, net.minecraft.nbt.CompoundTag nbt) throws IOException { ChunkPos nbtPos = nbt == null ? null : EntityStorage.readChunkPos(nbt); @@ -753,7 +750,7 @@ public class ServerLevel extends Level implements WorldGenLevel { // CraftBukkit end boolean flag2 = minecraftserver.forceSynchronousWrites(); DataFixer datafixer = minecraftserver.getFixerUpper(); - this.entityStorage = new EntityRegionFileStorage(convertable_conversionsession.getDimensionPath(resourcekey).resolve("entities"), flag2); // Paper - rewrite chunk system //EntityPersistentStorage entitypersistentstorage = new EntityStorage(this, convertable_conversionsession.getDimensionPath(resourcekey).resolve("entities"), datafixer, flag2, minecraftserver); + this.entityStorage = new EntityRegionFileStorage(this.getLevel().divinemcConfig.regionFormatName, this.getLevel().divinemcConfig.regionFormatLinearCompressionLevel, this.getLevel().divinemcConfig.linearCrashOnBrokenSymlink, convertable_conversionsession.getDimensionPath(resourcekey).resolve("entities"), flag2); // Paper - rewrite chunk system //EntityPersistentStorage entitypersistentstorage = new EntityStorage(this, convertable_conversionsession.getDimensionPath(resourcekey).resolve("entities"), datafixer, flag2, minecraftserver); // DivineMC - Implement Linear region // this.entityManager = new PersistentEntitySectionManager<>(Entity.class, new ServerLevel.EntityCallbacks(), entitypersistentstorage, this.entitySliceManager); // Paper // Paper - rewrite chunk system StructureTemplateManager structuretemplatemanager = minecraftserver.getStructureManager(); diff --git a/src/main/java/net/minecraft/util/worldupdate/WorldUpgrader.java b/src/main/java/net/minecraft/util/worldupdate/WorldUpgrader.java index f2a7cb6ebed7a4b4019a09af2a025f624f6fe9c9..3ddbded1a5f1dcb82e1309c3741bff09041c807c 100644 --- a/src/main/java/net/minecraft/util/worldupdate/WorldUpgrader.java +++ b/src/main/java/net/minecraft/util/worldupdate/WorldUpgrader.java @@ -8,6 +8,9 @@ import com.google.common.collect.UnmodifiableIterator; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.mojang.datafixers.DataFixer; import com.mojang.logging.LogUtils; +import gq.bxteam.divinemc.region.AbstractRegionFile; +import gq.bxteam.divinemc.region.AbstractRegionFileFactory; +import gq.bxteam.divinemc.region.RegionFileFormat; import it.unimi.dsi.fastutil.objects.Reference2FloatMap; import it.unimi.dsi.fastutil.objects.Reference2FloatMaps; import it.unimi.dsi.fastutil.objects.Reference2FloatOpenHashMap; @@ -37,7 +40,6 @@ import net.minecraft.world.level.ChunkPos; import net.minecraft.world.level.Level; import net.minecraft.world.level.chunk.ChunkGenerator; import net.minecraft.world.level.chunk.storage.ChunkStorage; -import net.minecraft.world.level.chunk.storage.RegionFile; import net.minecraft.world.level.dimension.LevelStem; import net.minecraft.world.level.storage.DimensionDataStorage; import net.minecraft.world.level.storage.LevelStorageSource; @@ -61,7 +63,7 @@ public class WorldUpgrader { private volatile int skipped; private final Reference2FloatMap> progressMap = Reference2FloatMaps.synchronize(new Reference2FloatOpenHashMap()); private volatile Component status = Component.translatable("optimizeWorld.stage.counting"); - public static final Pattern REGEX = Pattern.compile("^r\\.(-?[0-9]+)\\.(-?[0-9]+)\\.mca$"); + public static Pattern REGEX = Pattern.compile("^r\\.(-?[0-9]+)\\.(-?[0-9]+)\\.(linear | mca)$"); // DivineMC - Implement Linear region private final DimensionDataStorage overworldDataStorage; public WorldUpgrader(LevelStorageSource.LevelStorageAccess session, DataFixer dataFixer, Registry dimensionOptionsRegistry, boolean eraseCache) { @@ -116,7 +118,13 @@ public class WorldUpgrader { ResourceKey resourcekey1 = (ResourceKey) iterator1.next(); Path path = this.levelStorage.getDimensionPath(resourcekey1); - builder1.put(resourcekey1, new ChunkStorage(path.resolve("region"), this.dataFixer, true)); + // DivineMC start - Implement Linear region + String worldName = this.levelStorage.getLevelId(); + RegionFileFormat formatName = ((org.bukkit.craftbukkit.CraftWorld) org.bukkit.Bukkit.getWorld(worldName)).getHandle().divinemcConfig.regionFormatName; + int linearCompression = ((org.bukkit.craftbukkit.CraftWorld) org.bukkit.Bukkit.getWorld(worldName)).getHandle().divinemcConfig.regionFormatLinearCompressionLevel; + boolean linearCrashOnBrokenSymlink = ((org.bukkit.craftbukkit.CraftWorld) org.bukkit.Bukkit.getWorld(worldName)).getHandle().divinemcConfig.linearCrashOnBrokenSymlink; + builder1.put(resourcekey1, new ChunkStorage(formatName, linearCompression, linearCrashOnBrokenSymlink, path.resolve("region"), this.dataFixer, true)); + // DivineMC end } ImmutableMap, ChunkStorage> immutablemap1 = builder1.build(); @@ -235,7 +243,7 @@ public class WorldUpgrader { File file = this.levelStorage.getDimensionPath(world).toFile(); File file1 = new File(file, "region"); File[] afile = file1.listFiles((file2, s) -> { - return s.endsWith(".mca"); + return s.endsWith(".mca") || s.endsWith(".linear"); // DivineMC - Implement Linear region }); if (afile == null) { @@ -254,7 +262,11 @@ public class WorldUpgrader { int l = Integer.parseInt(matcher.group(2)) << 5; try { - RegionFile regionfile = new RegionFile(file2.toPath(), file1.toPath(), true); + // DivineMC start - Implement Linear region + String worldName = this.levelStorage.getLevelId(); + int linearCompression = ((org.bukkit.craftbukkit.CraftWorld) org.bukkit.Bukkit.getWorld(worldName)).getHandle().divinemcConfig.regionFormatLinearCompressionLevel; + AbstractRegionFile regionfile = AbstractRegionFileFactory.getAbstractRegionFile(linearCompression, file2.toPath(), file1.toPath(), true); + // DivineMC end try { for (int i1 = 0; i1 < 32; ++i1) { diff --git a/src/main/java/net/minecraft/world/entity/ai/village/poi/PoiManager.java b/src/main/java/net/minecraft/world/entity/ai/village/poi/PoiManager.java index 12a7aaeaa8b4b788b620b1985591c3b93253ccd5..ab1d0f6b7a4d29a073798d7782777c56c3ece530 100644 --- a/src/main/java/net/minecraft/world/entity/ai/village/poi/PoiManager.java +++ b/src/main/java/net/minecraft/world/entity/ai/village/poi/PoiManager.java @@ -2,10 +2,10 @@ package net.minecraft.world.entity.ai.village.poi; import com.mojang.datafixers.DataFixer; import com.mojang.datafixers.util.Pair; +import gq.bxteam.divinemc.region.RegionFileFormat; import it.unimi.dsi.fastutil.longs.Long2ByteMap; import it.unimi.dsi.fastutil.longs.Long2ByteOpenHashMap; -import it.unimi.dsi.fastutil.longs.LongOpenHashSet; -import it.unimi.dsi.fastutil.longs.LongSet; + import java.nio.file.Path; import java.util.Comparator; import java.util.List; @@ -57,8 +57,10 @@ public class PoiManager extends SectionStorage { // Paper end - rewrite chunk system - public PoiManager(Path path, DataFixer dataFixer, boolean dsync, RegistryAccess registryManager, LevelHeightAccessor world) { - super(path, PoiSection::codec, PoiSection::new, dataFixer, DataFixTypes.POI_CHUNK, dsync, registryManager, world); + // DivineMC start - Implement Linear region + public PoiManager(RegionFileFormat formatName, int linearCompression, boolean linearCrashOnBrokenSymlink, Path path, DataFixer dataFixer, boolean dsync, RegistryAccess registryManager, LevelHeightAccessor world) { // LinearPurpur + super(formatName, linearCompression, linearCrashOnBrokenSymlink, path, PoiSection::codec, PoiSection::new, dataFixer, DataFixTypes.POI_CHUNK, dsync, registryManager, world); + // DivineMC end this.world = (net.minecraft.server.level.ServerLevel)world; // Paper - rewrite chunk system } diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/ChunkStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/ChunkStorage.java index 8ebecb588058da174b0e0e19e54fcddfeeca1422..15658d929294f24ca8e1d2b9ef11b64c8deb1ed7 100644 --- a/src/main/java/net/minecraft/world/level/chunk/storage/ChunkStorage.java +++ b/src/main/java/net/minecraft/world/level/chunk/storage/ChunkStorage.java @@ -8,13 +8,14 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; import javax.annotation.Nullable; + +import gq.bxteam.divinemc.region.RegionFileFormat; import net.minecraft.SharedConstants; import net.minecraft.nbt.CompoundTag; import net.minecraft.nbt.NbtUtils; import net.minecraft.resources.ResourceKey; import net.minecraft.server.level.ServerChunkCache; import net.minecraft.server.level.ServerLevel; -import net.minecraft.util.datafix.DataFixTypes; import net.minecraft.world.level.ChunkPos; import net.minecraft.world.level.LevelAccessor; import net.minecraft.world.level.chunk.ChunkGenerator; @@ -37,11 +38,11 @@ public class ChunkStorage implements AutoCloseable { public final RegionFileStorage regionFileCache; // Paper end - async chunk loading - public ChunkStorage(Path directory, DataFixer dataFixer, boolean dsync) { + public ChunkStorage(RegionFileFormat format, int linearCompression, boolean linearCrashOnBrokenSymlink, Path directory, DataFixer dataFixer, boolean dsync) { // DivineMC - Implement Linear region this.fixerUpper = dataFixer; // Paper start - async chunk io // remove IO worker - this.regionFileCache = new RegionFileStorage(directory, dsync, true); // Paper - nuke IOWorker // Paper + this.regionFileCache = new RegionFileStorage(format, linearCompression, linearCrashOnBrokenSymlink, directory, dsync, true); // Paper - nuke IOWorker // Paper // DivineMC - Implement Linear region // Paper end - async chunk io } diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java index 9248769e6d357f6eec68945fd7700e79b2942c41..4d9dd194d9754103579c15884f799f051402b6fb 100644 --- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java +++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java @@ -20,13 +20,15 @@ import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.util.zip.InflaterInputStream; // Paper import javax.annotation.Nullable; + +import gq.bxteam.divinemc.region.AbstractRegionFile; import net.minecraft.Util; import net.minecraft.nbt.CompoundTag; // Paper import net.minecraft.nbt.NbtIo; // Paper import net.minecraft.world.level.ChunkPos; import org.slf4j.Logger; -public class RegionFile implements AutoCloseable { +public class RegionFile implements AutoCloseable, AbstractRegionFile { // DivineMC - Implement Linear region private static final Logger LOGGER = LogUtils.getLogger(); private static final int SECTOR_BYTES = 4096; @@ -50,6 +52,16 @@ public class RegionFile implements AutoCloseable { public final java.util.concurrent.locks.ReentrantLock fileLock = new java.util.concurrent.locks.ReentrantLock(); // Paper public final Path regionFile; // Paper + // DivineMC start - Implement Linear region + public Path getRegionFile() { + return this.regionFile; + } + + public java.util.concurrent.locks.ReentrantLock getFileLock() { + return this.fileLock; + } + // DivineMC end + // Paper start - try to recover from RegionFile header corruption private static long roundToSectors(long bytes) { long sectors = bytes >>> 12; // 4096 = 2^12 @@ -128,7 +140,7 @@ public class RegionFile implements AutoCloseable { } // note: only call for CHUNK regionfiles - boolean recalculateHeader() throws IOException { + public boolean recalculateHeader() throws IOException { // DivineMC - Implement Linear region if (!this.canRecalcHeader) { return false; } @@ -954,10 +966,11 @@ public class RegionFile implements AutoCloseable { private static int getChunkIndex(int x, int z) { return (x & 31) + (z & 31) * 32; } - synchronized boolean isOversized(int x, int z) { + public synchronized boolean isOversized(int x, int z) { // DivineMC - Implement Linear region return this.oversized[getChunkIndex(x, z)] == 1; } - synchronized void setOversized(int x, int z, boolean oversized) throws IOException { + + public synchronized void setOversized(int x, int z, boolean oversized) throws IOException { // DivineMC - Implement Linear region final int offset = getChunkIndex(x, z); boolean previous = this.oversized[offset] == 1; this.oversized[offset] = (byte) (oversized ? 1 : 0); @@ -996,7 +1009,7 @@ public class RegionFile implements AutoCloseable { return this.regionFile.getParent().resolve(this.regionFile.getFileName().toString().replaceAll("\\.mca$", "") + "_oversized_" + x + "_" + z + ".nbt"); } - synchronized CompoundTag getOversizedData(int x, int z) throws IOException { + public synchronized CompoundTag getOversizedData(int x, int z) throws IOException { // DivineMC - Implement Linear region Path file = getOversizedFile(x, z); try (DataInputStream out = new DataInputStream(new java.io.BufferedInputStream(new InflaterInputStream(Files.newInputStream(file))))) { return NbtIo.read((java.io.DataInput) out); diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java index 6eaeb2db0da59611501f2b1a63b5b48816a0ba48..0e61161d7d8d714bb7a7c3f68e018e867c6161a0 100644 --- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java +++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java @@ -1,5 +1,8 @@ package net.minecraft.world.level.chunk.storage; +import gq.bxteam.divinemc.region.AbstractRegionFile; +import gq.bxteam.divinemc.region.AbstractRegionFileFactory; +import gq.bxteam.divinemc.region.RegionFileFormat; import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap; import it.unimi.dsi.fastutil.objects.ObjectIterator; import java.io.DataInput; @@ -18,12 +21,17 @@ import net.minecraft.util.ExceptionCollector; import net.minecraft.world.level.ChunkPos; public class RegionFileStorage implements AutoCloseable { - + private static final org.slf4j.Logger LOGGER = com.mojang.logging.LogUtils.getLogger(); // DivineMC - Implement Linear region public static final String ANVIL_EXTENSION = ".mca"; private static final int MAX_CACHE_SIZE = 256; - public final Long2ObjectLinkedOpenHashMap regionCache = new Long2ObjectLinkedOpenHashMap(); + public final Long2ObjectLinkedOpenHashMap regionCache = new Long2ObjectLinkedOpenHashMap(); // DivineMC - Implement Linear region private final Path folder; private final boolean sync; + // DivineMC start - Implement Linear region + public final RegionFileFormat format; + public final int linearCompression; + public final boolean linearCrashOnBrokenSymlink; + // DivineMC end private final boolean isChunkData; // Paper // Paper start - cache regionfile does not exist state @@ -55,11 +63,17 @@ public class RegionFileStorage implements AutoCloseable { } // Paper end - cache regionfile does not exist state - protected RegionFileStorage(Path directory, boolean dsync) { // Paper - protected constructor + protected RegionFileStorage(RegionFileFormat format, int linearCompression, boolean linearCrashOnBrokenSymlink, Path directory, boolean dsync) { // Paper - protected constructor // DivineMC - Implement Linear region // Paper start - add isChunkData param - this(directory, dsync, false); + this(format, linearCompression, linearCrashOnBrokenSymlink, directory, dsync, false); // DivineMC - Implement Linear region } - RegionFileStorage(Path directory, boolean dsync, boolean isChunkData) { + + RegionFileStorage(RegionFileFormat format, int linearCompression, boolean linearCrashOnBrokenSymlink, Path directory, boolean dsync, boolean isChunkData) { // DivineMC - Implement Linear region + // DivineMC start - Implement Linear region + this.format = format; + this.linearCompression = linearCompression; + this.linearCrashOnBrokenSymlink = linearCrashOnBrokenSymlink; + // DivineMC end this.isChunkData = isChunkData; // Paper end - add isChunkData param this.folder = directory; @@ -70,7 +84,7 @@ public class RegionFileStorage implements AutoCloseable { @Nullable public static ChunkPos getRegionFileCoordinates(Path file) { String fileName = file.getFileName().toString(); - if (!fileName.startsWith("r.") || !fileName.endsWith(".mca")) { + if (!fileName.startsWith("r.") || !fileName.endsWith(".mca") || !fileName.endsWith(".linear")) { // DivineMC - Implement Linear region return null; } @@ -89,30 +103,45 @@ public class RegionFileStorage implements AutoCloseable { return null; } } - - public synchronized RegionFile getRegionFileIfLoaded(ChunkPos chunkcoordintpair) { + + public synchronized AbstractRegionFile getRegionFileIfLoaded(ChunkPos chunkcoordintpair) { // DivineMC - Implement Linear region return this.regionCache.getAndMoveToFirst(ChunkPos.asLong(chunkcoordintpair.getRegionX(), chunkcoordintpair.getRegionZ())); } public synchronized boolean chunkExists(ChunkPos pos) throws IOException { - RegionFile regionfile = getRegionFile(pos, true); + AbstractRegionFile regionfile = getRegionFile(pos, true); // DivineMC - Implement Linear region return regionfile != null ? regionfile.hasChunk(pos) : false; } - public synchronized RegionFile getRegionFile(ChunkPos chunkcoordintpair, boolean existingOnly) throws IOException { // CraftBukkit + // DivineMC start - Implement Linear region + private void guardAgainstBrokenSymlinks(Path path) throws IOException { + if (!linearCrashOnBrokenSymlink) return; + if (!this.format.equals("LINEAR")) return; + if (!java.nio.file.Files.isSymbolicLink(path)) return; + Path link = java.nio.file.Files.readSymbolicLink(path); + if (!java.nio.file.Files.exists(link) || !java.nio.file.Files.isReadable(link)) { + LOGGER.error("Linear region file {} is a broken symbolic link, crashing to prevent data loss", path); + net.minecraft.server.MinecraftServer.getServer().halt(false); + throw new IOException("Linear region file " + path + " is a broken symbolic link, crashing to prevent data loss"); + } + } + // DivineMC end + + public synchronized AbstractRegionFile getRegionFile(ChunkPos chunkcoordintpair, boolean existingOnly) throws IOException { // CraftBukkit // DivineMC - Implement Linear region return this.getRegionFile(chunkcoordintpair, existingOnly, false); } - public synchronized RegionFile getRegionFile(ChunkPos chunkcoordintpair, boolean existingOnly, boolean lock) throws IOException { + + public synchronized AbstractRegionFile getRegionFile(ChunkPos chunkcoordintpair, boolean existingOnly, boolean lock) throws IOException { // DivineMC - Implement Linear region // Paper end long i = ChunkPos.asLong(chunkcoordintpair.getRegionX(), chunkcoordintpair.getRegionZ()); final long regionPos = i; // Paper - OBFHELPER - RegionFile regionfile = (RegionFile) this.regionCache.getAndMoveToFirst(i); + AbstractRegionFile regionfile = this.regionCache.getAndMoveToFirst(i); // DivineMC - Implement Linear region if (regionfile != null) { // Paper start if (lock) { // must be in this synchronized block - regionfile.fileLock.lock(); + regionfile.getFileLock().lock(); // DivineMC - Implement Linear region } // Paper end return regionfile; @@ -123,28 +152,45 @@ public class RegionFileStorage implements AutoCloseable { } // Paper end - cache regionfile does not exist state if (this.regionCache.size() >= io.papermc.paper.configuration.GlobalConfiguration.get().misc.regionFileCacheSize) { // Paper - configurable - ((RegionFile) this.regionCache.removeLast()).close(); + this.regionCache.removeLast().close(); // DivineMC - Implement Linear region } // Paper - only create directory if not existing only - moved down Path path = this.folder; int j = chunkcoordintpair.getRegionX(); - Path path1 = path.resolve("r." + j + "." + chunkcoordintpair.getRegionZ() + ".mca"); // Paper - diff on change - if (existingOnly && !java.nio.file.Files.exists(path1)) { // Paper start - cache regionfile does not exist state - this.markNonExisting(regionPos); - return null; // CraftBukkit + // DivineMC start - Implement Linear region + Path path1; + if (existingOnly) { + Path anvil = path.resolve("r." + j + "." + chunkcoordintpair.getRegionZ() + ".mca"); + Path linear = path.resolve("r." + j + "." + chunkcoordintpair.getRegionZ() + ".linear"); + guardAgainstBrokenSymlinks(linear); + if (java.nio.file.Files.exists(anvil)) path1 = anvil; + else if (java.nio.file.Files.exists(linear)) path1 = linear; + else { + this.markNonExisting(regionPos); + return null; + } + // DivineMC end } else { + // DivineMC start - Implement Linear region + String extension = switch (this.format) { + case LINEAR -> "linear"; + default -> "mca"; + }; + path1 = path.resolve("r." + j + "." + chunkcoordintpair.getRegionZ() + "." + extension); + guardAgainstBrokenSymlinks(path1); + // DivineMC end this.createRegionFile(regionPos); } // Paper end - cache regionfile does not exist state FileUtil.createDirectoriesSafe(this.folder); // Paper - only create directory if not existing only - moved from above - RegionFile regionfile1 = new RegionFile(path1, this.folder, this.sync, this.isChunkData); // Paper - allow for chunk regionfiles to regen header + AbstractRegionFile regionfile1 = AbstractRegionFileFactory.getAbstractRegionFile(this.linearCompression, path1, this.folder, this.sync, this.isChunkData); // Paper - allow for chunk regionfiles to regen header // DivineMC - Implement Linear region this.regionCache.putAndMoveToFirst(i, regionfile1); // Paper start if (lock) { // must be in this synchronized block - regionfile1.fileLock.lock(); + regionfile1.getFileLock().lock(); // DivineMC - Implement Linear region } // Paper end return regionfile1; @@ -172,7 +218,7 @@ public class RegionFileStorage implements AutoCloseable { } - private static CompoundTag readOversizedChunk(RegionFile regionfile, ChunkPos chunkCoordinate) throws IOException { + private static CompoundTag readOversizedChunk(AbstractRegionFile regionfile, ChunkPos chunkCoordinate) throws IOException { // DivineMC - Implement Linear region synchronized (regionfile) { try (DataInputStream datainputstream = regionfile.getChunkDataInputStream(chunkCoordinate)) { CompoundTag oversizedData = regionfile.getOversizedData(chunkCoordinate.x, chunkCoordinate.z); @@ -219,14 +265,15 @@ public class RegionFileStorage implements AutoCloseable { @Nullable public CompoundTag read(ChunkPos pos) throws IOException { // CraftBukkit start - SPIGOT-5680: There's no good reason to preemptively create files on read, save that for writing - RegionFile regionfile = this.getRegionFile(pos, true, true); // Paper + AbstractRegionFile regionfile = this.getRegionFile(pos, true, true); // Paper // DivineMC - Implement Linear region if (regionfile == null) { return null; } // Paper start - Add regionfile parameter return this.read(pos, regionfile); } - public CompoundTag read(ChunkPos pos, RegionFile regionfile) throws IOException { + + public CompoundTag read(ChunkPos pos, AbstractRegionFile regionfile) throws IOException { // DivineMC - Implement Linear region // We add the regionfile parameter to avoid the potential deadlock (on fileLock) if we went back to obtain a regionfile // if we decide to re-read // Paper end @@ -236,7 +283,7 @@ public class RegionFileStorage implements AutoCloseable { // Paper start if (regionfile.isOversized(pos.x, pos.z)) { - printOversizedLog("Loading Oversized Chunk!", regionfile.regionFile, pos.x, pos.z); + printOversizedLog("Loading Oversized Chunk!", regionfile.getRegionFile(), pos.x, pos.z); // DivineMC - Implement Linear region return readOversizedChunk(regionfile, pos); } // Paper end @@ -250,12 +297,12 @@ public class RegionFileStorage implements AutoCloseable { if (this.isChunkData) { ChunkPos chunkPos = ChunkSerializer.getChunkCoordinate(nbttagcompound); if (!chunkPos.equals(pos)) { - net.minecraft.server.MinecraftServer.LOGGER.error("Attempting to read chunk data at " + pos + " but got chunk data for " + chunkPos + " instead! Attempting regionfile recalculation for regionfile " + regionfile.regionFile.toAbsolutePath()); + net.minecraft.server.MinecraftServer.LOGGER.error("Attempting to read chunk data at " + pos + " but got chunk data for " + chunkPos + " instead! Attempting regionfile recalculation for regionfile " + regionfile.getRegionFile().toAbsolutePath()); // DivineMC - Implement Linear region if (regionfile.recalculateHeader()) { - regionfile.fileLock.lock(); // otherwise we will unlock twice and only lock once. + regionfile.getFileLock().lock(); // otherwise we will unlock twice and only lock once. // DivineMC - Implement Linear region return this.read(pos, regionfile); } - net.minecraft.server.MinecraftServer.LOGGER.error("Can't recalculate regionfile header, regenerating chunk " + pos + " for " + regionfile.regionFile.toAbsolutePath()); + net.minecraft.server.MinecraftServer.LOGGER.error("Can't recalculate regionfile header, regenerating chunk " + pos + " for " + regionfile.getRegionFile().toAbsolutePath()); // DivineMC - Implement Linear region return null; } } @@ -289,13 +336,13 @@ public class RegionFileStorage implements AutoCloseable { return nbttagcompound; } finally { // Paper start - regionfile.fileLock.unlock(); + regionfile.getFileLock().unlock(); // DivineMC - Implement Linear region } // Paper end } public void scanChunk(ChunkPos chunkPos, StreamTagVisitor scanner) throws IOException { // CraftBukkit start - SPIGOT-5680: There's no good reason to preemptively create files on read, save that for writing - RegionFile regionfile = this.getRegionFile(chunkPos, true); + AbstractRegionFile regionfile = this.getRegionFile(chunkPos, true); // DivineMC - Implement Linear region if (regionfile == null) { return; } @@ -325,7 +372,7 @@ public class RegionFileStorage implements AutoCloseable { } protected void write(ChunkPos pos, @Nullable CompoundTag nbt) throws IOException { - RegionFile regionfile = this.getRegionFile(pos, nbt == null, true); // CraftBukkit // Paper // Paper start - rewrite chunk system + AbstractRegionFile regionfile = this.getRegionFile(pos, nbt == null, true); // CraftBukkit // Paper // Paper start - rewrite chunk system // DivineMC - Implement Linear region if (nbt == null && regionfile == null) { return; } @@ -375,7 +422,7 @@ public class RegionFileStorage implements AutoCloseable { } // Paper end } finally { // Paper start - regionfile.fileLock.unlock(); + regionfile.getFileLock().unlock(); // DivineMC - Implement Linear region } // Paper end } @@ -384,7 +431,7 @@ public class RegionFileStorage implements AutoCloseable { ObjectIterator objectiterator = this.regionCache.values().iterator(); while (objectiterator.hasNext()) { - RegionFile regionfile = (RegionFile) objectiterator.next(); + AbstractRegionFile regionfile = (AbstractRegionFile) objectiterator.next(); // DivineMC - Implement Linear region try { regionfile.close(); @@ -400,7 +447,7 @@ public class RegionFileStorage implements AutoCloseable { ObjectIterator objectiterator = this.regionCache.values().iterator(); while (objectiterator.hasNext()) { - RegionFile regionfile = (RegionFile) objectiterator.next(); + AbstractRegionFile regionfile = (AbstractRegionFile) objectiterator.next(); // DivineMC - Implement Linear region regionfile.flush(); } diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/SectionStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/SectionStorage.java index 4aac1979cf57300825a999c876fcf24d3170e68e..b81636c1aa5b39e1696fdf30a7e2b270ad688a58 100644 --- a/src/main/java/net/minecraft/world/level/chunk/storage/SectionStorage.java +++ b/src/main/java/net/minecraft/world/level/chunk/storage/SectionStorage.java @@ -9,6 +9,7 @@ import com.mojang.serialization.DataResult; import com.mojang.serialization.Dynamic; import com.mojang.serialization.DynamicOps; import com.mojang.serialization.OptionalDynamic; +import gq.bxteam.divinemc.region.RegionFileFormat; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; import it.unimi.dsi.fastutil.longs.LongLinkedOpenHashSet; @@ -17,7 +18,6 @@ import java.nio.file.Path; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; import java.util.function.BooleanSupplier; import java.util.function.Function; import javax.annotation.Nullable; @@ -47,8 +47,10 @@ public class SectionStorage extends RegionFileStorage implements AutoCloseabl public final RegistryAccess registryAccess; // Paper - rewrite chunk system protected final LevelHeightAccessor levelHeightAccessor; - public SectionStorage(Path path, Function> codecFactory, Function factory, DataFixer dataFixer, DataFixTypes dataFixTypes, boolean dsync, RegistryAccess dynamicRegistryManager, LevelHeightAccessor world) { - super(path, dsync); // Paper - remove mojang I/O thread + // DivineMC start - Implement Linear region + public SectionStorage(RegionFileFormat format, int linearCompression, boolean linearCrashOnBrokenSymlink, Path path, Function> codecFactory, Function factory, DataFixer dataFixer, DataFixTypes dataFixTypes, boolean dsync, RegistryAccess dynamicRegistryManager, LevelHeightAccessor world) { + super(format, linearCompression, linearCrashOnBrokenSymlink, path, dsync); // Paper - remove mojang I/O thread + // DivineMC end this.codec = codecFactory; this.factory = factory; this.fixerUpper = dataFixer; diff --git a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java index f6c6cd92e1eff044abefa6ca74477d361f4434ec..e24498294f44877bcf8fc97cea9e3ca1a4f9200d 100644 --- a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java +++ b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java @@ -5,7 +5,7 @@ import com.google.common.base.Predicates; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.mojang.datafixers.util.Pair; -import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap; +import gq.bxteam.divinemc.region.AbstractRegionFile; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; import java.io.File; @@ -20,7 +20,6 @@ import java.util.Objects; import java.util.Random; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ExecutionException; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -120,7 +119,6 @@ import org.bukkit.entity.TippedArrow; import org.bukkit.entity.Trident; import org.bukkit.event.entity.CreatureSpawnEvent.SpawnReason; import org.bukkit.event.weather.LightningStrikeEvent; -import org.bukkit.event.world.SpawnChangeEvent; import org.bukkit.event.world.TimeSkipEvent; import org.bukkit.generator.BiomeProvider; import org.bukkit.generator.BlockPopulator; @@ -567,7 +565,7 @@ public class CraftWorld extends CraftRegionAccessor implements World { return true; } - net.minecraft.world.level.chunk.storage.RegionFile file; + AbstractRegionFile file; // DivineMC - Implement Linear region try { file = world.getChunkSource().chunkMap.regionFileCache.getRegionFile(chunkPos, false); } catch (java.io.IOException ex) {