From 1dca502a90794d6da78e4b7274a308c9f249bf1f Mon Sep 17 00:00:00 2001 From: Julian Krings Date: Wed, 27 Nov 2024 19:45:08 +0100 Subject: [PATCH] add more safety to mantle --- .../iris/core/commands/CommandDeveloper.java | 9 +- .../volmit/iris/engine/IrisWorldManager.java | 2 +- .../iris/util/io/CountingDataInputStream.java | 87 ++++++++++++++ .../com/volmit/iris/util/mantle/Mantle.java | 3 - .../volmit/iris/util/mantle/MantleChunk.java | 37 ++++-- .../iris/util/mantle/TectonicPlate.java | 106 ++++++++++++------ .../com/volmit/iris/util/matter/Matter.java | 43 +++++-- 7 files changed, 228 insertions(+), 59 deletions(-) create mode 100644 core/src/main/java/com/volmit/iris/util/io/CountingDataInputStream.java diff --git a/core/src/main/java/com/volmit/iris/core/commands/CommandDeveloper.java b/core/src/main/java/com/volmit/iris/core/commands/CommandDeveloper.java index cb43a29d2..7d4d2042e 100644 --- a/core/src/main/java/com/volmit/iris/core/commands/CommandDeveloper.java +++ b/core/src/main/java/com/volmit/iris/core/commands/CommandDeveloper.java @@ -42,6 +42,7 @@ import com.volmit.iris.util.decree.annotations.Decree; import com.volmit.iris.util.decree.annotations.Param; import com.volmit.iris.util.format.C; import com.volmit.iris.util.format.Form; +import com.volmit.iris.util.io.CountingDataInputStream; import com.volmit.iris.util.io.IO; import com.volmit.iris.util.mantle.TectonicPlate; import com.volmit.iris.util.math.Spiraler; @@ -262,7 +263,7 @@ public class CommandDeveloper implements DecreeExecutor { VolmitSender sender = sender(); service.submit(() -> { try { - DataInputStream raw = new DataInputStream(new FileInputStream(file)); + CountingDataInputStream raw = CountingDataInputStream.wrap(new FileInputStream(file)); TectonicPlate plate = new TectonicPlate(height, raw); raw.close(); @@ -281,7 +282,7 @@ public class CommandDeveloper implements DecreeExecutor { if (size == 0) size = tmp.length(); start = System.currentTimeMillis(); - DataInputStream din = createInput(tmp, algorithm); + CountingDataInputStream din = createInput(tmp, algorithm); new TectonicPlate(height, din); din.close(); d2 += System.currentTimeMillis() - start; @@ -301,10 +302,10 @@ public class CommandDeveloper implements DecreeExecutor { } } - private DataInputStream createInput(File file, String algorithm) throws Throwable { + private CountingDataInputStream createInput(File file, String algorithm) throws Throwable { FileInputStream in = new FileInputStream(file); - return new DataInputStream(switch (algorithm) { + return CountingDataInputStream.wrap(switch (algorithm) { case "gzip" -> new GZIPInputStream(in); case "lz4f" -> new LZ4FrameInputStream(in); case "lz4b" -> new LZ4BlockInputStream(in); diff --git a/core/src/main/java/com/volmit/iris/engine/IrisWorldManager.java b/core/src/main/java/com/volmit/iris/engine/IrisWorldManager.java index 075e8660d..11f7e0d55 100644 --- a/core/src/main/java/com/volmit/iris/engine/IrisWorldManager.java +++ b/core/src/main/java/com/volmit/iris/engine/IrisWorldManager.java @@ -459,7 +459,7 @@ public class IrisWorldManager extends EngineAssignedWorldManager { IrisEngineData ed = getEngine().getEngineData(); IrisEngineSpawnerCooldown cd = null; - for (IrisEngineSpawnerCooldown j : ed.getSpawnerCooldowns()) { + for (IrisEngineSpawnerCooldown j : ed.getSpawnerCooldowns().copy()) { if (j.getSpawner().equals(i.getLoadKey())) { cd = j; } diff --git a/core/src/main/java/com/volmit/iris/util/io/CountingDataInputStream.java b/core/src/main/java/com/volmit/iris/util/io/CountingDataInputStream.java new file mode 100644 index 000000000..6239518b1 --- /dev/null +++ b/core/src/main/java/com/volmit/iris/util/io/CountingDataInputStream.java @@ -0,0 +1,87 @@ +package com.volmit.iris.util.io; + +import lombok.RequiredArgsConstructor; +import org.jetbrains.annotations.NotNull; + +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; + +public class CountingDataInputStream extends DataInputStream { + private final Counter counter; + + private CountingDataInputStream(@NotNull InputStream in) { + super(in); + if (!(in instanceof Counter c)) + throw new IllegalArgumentException("Underlying stream must be a Counter"); + this.counter = c; + } + + public static CountingDataInputStream wrap(@NotNull InputStream in) { + return new CountingDataInputStream(new Counter(in)); + } + + public long count() { + return counter.count; + } + + public void skipTo(long target) throws IOException { + skipNBytes(Math.max(target - counter.count, 0)); + } + + @RequiredArgsConstructor + private static class Counter extends InputStream { + private final InputStream in; + private long count; + private long mark = -1; + private int markLimit = 0; + + @Override + public int read() throws IOException { + int i = in.read(); + if (i != -1) count(1); + return i; + } + + @Override + public int read(@NotNull byte[] b, int off, int len) throws IOException { + int i = in.read(b, off, len); + count(i); + return i; + } + + private void count(int i) { + count += i; + if (mark == -1) + return; + + markLimit -= i; + if (markLimit <= 0) + mark = -1; + } + + @Override + public boolean markSupported() { + return in.markSupported(); + } + + @Override + public synchronized void mark(int readlimit) { + if (!in.markSupported()) return; + in.mark(readlimit); + mark = count; + markLimit = readlimit; + } + + @Override + public synchronized void reset() throws IOException { + in.reset(); + count = mark; + } + + @Override + public void close() throws IOException { + in.close(); + } + } +} diff --git a/core/src/main/java/com/volmit/iris/util/mantle/Mantle.java b/core/src/main/java/com/volmit/iris/util/mantle/Mantle.java index b89804537..4ebe1d1c3 100644 --- a/core/src/main/java/com/volmit/iris/util/mantle/Mantle.java +++ b/core/src/main/java/com/volmit/iris/util/mantle/Mantle.java @@ -567,9 +567,6 @@ public class Mantle { } File file = fileForRegion(dataFolder, x, z); - if (!file.exists()) - file = new File(dataFolder, file.getName().substring(".lz4b".length())); - if (file.exists()) { try { Iris.addPanic("reading.tectonic-plate", file.getAbsolutePath()); diff --git a/core/src/main/java/com/volmit/iris/util/mantle/MantleChunk.java b/core/src/main/java/com/volmit/iris/util/mantle/MantleChunk.java index e73e7d68e..816896663 100644 --- a/core/src/main/java/com/volmit/iris/util/mantle/MantleChunk.java +++ b/core/src/main/java/com/volmit/iris/util/mantle/MantleChunk.java @@ -21,12 +21,13 @@ package com.volmit.iris.util.mantle; import com.volmit.iris.Iris; import com.volmit.iris.util.documentation.ChunkCoordinates; import com.volmit.iris.util.function.Consumer4; +import com.volmit.iris.util.io.CountingDataInputStream; import com.volmit.iris.util.matter.IrisMatter; import com.volmit.iris.util.matter.Matter; import com.volmit.iris.util.matter.MatterSlice; import lombok.Getter; -import java.io.DataInputStream; +import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.IOException; import java.util.concurrent.atomic.AtomicIntegerArray; @@ -69,7 +70,7 @@ public class MantleChunk { * @throws IOException shit happens * @throws ClassNotFoundException shit happens */ - public MantleChunk(int sectionHeight, DataInputStream din) throws IOException, ClassNotFoundException { + public MantleChunk(int sectionHeight, CountingDataInputStream din) throws IOException { this(sectionHeight, din.readByte(), din.readByte()); int s = din.readByte(); @@ -79,8 +80,23 @@ public class MantleChunk { for (int i = 0; i < s; i++) { Iris.addPanic("read.section", "Section[" + i + "]"); - if (din.readBoolean()) { + long size = din.readInt(); + if (size == 0) continue; + long start = din.count(); + + try { sections.set(i, Matter.readDin(din)); + } catch (IOException e) { + long end = start + size; + Iris.error("Failed to read chunk section, skipping it."); + Iris.addPanic("read.byte.range", start + " " + end); + Iris.addPanic("read.byte.current", din.count() + ""); + Iris.reportError(e); + e.printStackTrace(); + Iris.panic(); + + din.skipTo(end); + TectonicPlate.addError(); } } } @@ -174,15 +190,22 @@ public class MantleChunk { dos.writeBoolean(flags.get(i) == 1); } + var bytes = new ByteArrayOutputStream(8192); + var sub = new DataOutputStream(bytes); for (int i = 0; i < sections.length(); i++) { trimSlice(i); if (exists(i)) { - dos.writeBoolean(true); - Matter matter = get(i); - matter.writeDos(dos); + try { + Matter matter = get(i); + matter.writeDos(sub); + dos.writeInt(bytes.size()); + bytes.writeTo(dos); + } finally { + bytes.reset(); + } } else { - dos.writeBoolean(false); + dos.writeInt(0); } } } diff --git a/core/src/main/java/com/volmit/iris/util/mantle/TectonicPlate.java b/core/src/main/java/com/volmit/iris/util/mantle/TectonicPlate.java index 044e1f745..f2bf590d6 100644 --- a/core/src/main/java/com/volmit/iris/util/mantle/TectonicPlate.java +++ b/core/src/main/java/com/volmit/iris/util/mantle/TectonicPlate.java @@ -21,26 +21,31 @@ package com.volmit.iris.util.mantle; import com.volmit.iris.Iris; import com.volmit.iris.engine.EnginePanic; import com.volmit.iris.engine.data.cache.Cache; +import com.volmit.iris.util.collection.KSet; import com.volmit.iris.util.documentation.ChunkCoordinates; import com.volmit.iris.util.format.C; import com.volmit.iris.util.format.Form; +import com.volmit.iris.util.io.CountingDataInputStream; import com.volmit.iris.util.scheduling.PrecisionStopwatch; import lombok.Getter; import net.jpountz.lz4.LZ4BlockInputStream; import net.jpountz.lz4.LZ4BlockOutputStream; -import net.jpountz.lz4.LZ4FrameInputStream; -import net.jpountz.lz4.LZ4FrameOutputStream; import java.io.*; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.util.concurrent.atomic.AtomicReferenceArray; -import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; /** * Tectonic Plates are essentially representations of regions in minecraft. * Tectonic Plates are fully atomic & thread safe */ public class TectonicPlate { + private static final KSet errors = new KSet<>(); + private final int sectionHeight; private final AtomicReferenceArray chunks; @@ -68,33 +73,58 @@ public class TectonicPlate { * @param worldHeight the height of the world * @param din the data input * @throws IOException shit happens yo - * @throws ClassNotFoundException real shit bro */ - public TectonicPlate(int worldHeight, DataInputStream din) throws IOException, ClassNotFoundException { + public TectonicPlate(int worldHeight, CountingDataInputStream din) throws IOException { this(worldHeight, din.readInt(), din.readInt()); + if (!din.markSupported()) + throw new IOException("Mark not supported!"); + for (int i = 0; i < chunks.length(); i++) { - if (din.readBoolean()) { + long size = din.readInt(); + if (size == 0) continue; + long start = din.count(); + + try { Iris.addPanic("read-chunk", "Chunk[" + i + "]"); chunks.set(i, new MantleChunk(sectionHeight, din)); EnginePanic.saveLast(); + } catch (Throwable e) { + long end = start + size; + Iris.error("Failed to read chunk, creating a new chunk instead."); + Iris.addPanic("read.byte.range", start + " " + end); + Iris.addPanic("read.byte.current", din.count() + ""); + Iris.reportError(e); + e.printStackTrace(); + Iris.panic(); + + din.skipTo(end); + TectonicPlate.addError(); } } } - public static TectonicPlate read(int worldHeight, File file) throws IOException, ClassNotFoundException { - FileInputStream fin = new FileInputStream(file); - DataInputStream din; - if (file.getName().endsWith("ttp.lz4b")) { - LZ4BlockInputStream lz4 = new LZ4BlockInputStream(fin); - din = new DataInputStream(new BufferedInputStream(lz4)); - } else { - GZIPInputStream gzi = new GZIPInputStream(fin); - din = new DataInputStream(new BufferedInputStream(gzi)); - } - TectonicPlate p = new TectonicPlate(worldHeight, din); - din.close(); + public static TectonicPlate read(int worldHeight, File file) throws IOException { + try (FileChannel fc = FileChannel.open(file.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.SYNC)) { + fc.lock(); - return p; + InputStream fin = Channels.newInputStream(fc); + LZ4BlockInputStream lz4 = new LZ4BlockInputStream(fin); + BufferedInputStream bis = new BufferedInputStream(lz4); + try (CountingDataInputStream din = CountingDataInputStream.wrap(bis)) { + return new TectonicPlate(worldHeight, din); + } + } finally { + if (errors.remove(Thread.currentThread())) { + File dump = Iris.instance.getDataFolder("dump", file.getName() + ".bin"); + try (FileChannel fc = FileChannel.open(file.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.SYNC)) { + fc.lock(); + + InputStream fin = Channels.newInputStream(fc); + LZ4BlockInputStream lz4 = new LZ4BlockInputStream(fin); + Files.copy(lz4, dump.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + } + } } /** @@ -173,18 +203,15 @@ public class TectonicPlate { */ public void write(File file) throws IOException { PrecisionStopwatch p = PrecisionStopwatch.start(); - FileOutputStream fos = new FileOutputStream(file); - DataOutputStream dos; - if (file.getName().endsWith("ttp.lz4b")) { - LZ4BlockOutputStream lz4 = new LZ4BlockOutputStream(fos); - dos = new DataOutputStream(lz4); - } else { - GZIPOutputStream gzo = new GZIPOutputStream(fos); - dos = new DataOutputStream(gzo); + try (FileChannel fc = FileChannel.open(file.toPath(), StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.SYNC)) { + fc.lock(); + + OutputStream fos = Channels.newOutputStream(fc); + try (DataOutputStream dos = new DataOutputStream(new LZ4BlockOutputStream(fos))) { + write(dos); + Iris.debug("Saved Tectonic Plate " + C.DARK_GREEN + file.getName().split("\\Q.\\E")[0] + C.RED + " in " + Form.duration(p.getMilliseconds(), 2)); + } } - write(dos); - dos.close(); - Iris.debug("Saved Tectonic Plate " + C.DARK_GREEN + file.getName().split("\\Q.\\E")[0] + C.RED + " in " + Form.duration(p.getMilliseconds(), 2)); } /** @@ -197,15 +224,26 @@ public class TectonicPlate { dos.writeInt(x); dos.writeInt(z); + var bytes = new ByteArrayOutputStream(8192); + var sub = new DataOutputStream(bytes); for (int i = 0; i < chunks.length(); i++) { MantleChunk chunk = chunks.get(i); if (chunk != null) { - dos.writeBoolean(true); - chunk.write(dos); + try { + chunk.write(sub); + dos.writeInt(bytes.size()); + bytes.writeTo(dos); + } finally { + bytes.reset(); + } } else { - dos.writeBoolean(false); + dos.writeInt(0); } } } + + public static void addError() { + errors.add(Thread.currentThread()); + } } diff --git a/core/src/main/java/com/volmit/iris/util/matter/Matter.java b/core/src/main/java/com/volmit/iris/util/matter/Matter.java index 9c07feaf6..c04fe2485 100644 --- a/core/src/main/java/com/volmit/iris/util/matter/Matter.java +++ b/core/src/main/java/com/volmit/iris/util/matter/Matter.java @@ -23,6 +23,8 @@ import com.volmit.iris.engine.object.IrisObject; import com.volmit.iris.engine.object.IrisPosition; import com.volmit.iris.util.collection.KSet; import com.volmit.iris.util.hunk.Hunk; +import com.volmit.iris.util.io.CountingDataInputStream; +import com.volmit.iris.util.mantle.TectonicPlate; import com.volmit.iris.util.math.BlockPosition; import org.bukkit.World; import org.bukkit.block.data.BlockData; @@ -96,18 +98,18 @@ public interface Matter { return m; } - static Matter read(File f) throws IOException, ClassNotFoundException { + static Matter read(File f) throws IOException { FileInputStream in = new FileInputStream(f); Matter m = read(in); in.close(); return m; } - static Matter read(InputStream in) throws IOException, ClassNotFoundException { + static Matter read(InputStream in) throws IOException { return read(in, (b) -> new IrisMatter(b.getX(), b.getY(), b.getZ())); } - static Matter readDin(DataInputStream in) throws IOException, ClassNotFoundException { + static Matter readDin(CountingDataInputStream in) throws IOException { return readDin(in, (b) -> new IrisMatter(b.getX(), b.getY(), b.getZ())); } @@ -120,11 +122,11 @@ public interface Matter { * @return the matter object * @throws IOException shit happens yo */ - static Matter read(InputStream in, Function matterFactory) throws IOException, ClassNotFoundException { - return readDin(new DataInputStream(in), matterFactory); + static Matter read(InputStream in, Function matterFactory) throws IOException { + return readDin(CountingDataInputStream.wrap(in), matterFactory); } - static Matter readDin(DataInputStream din, Function matterFactory) throws IOException, ClassNotFoundException { + static Matter readDin(CountingDataInputStream din, Function matterFactory) throws IOException { Matter matter = matterFactory.apply(new BlockPosition( din.readInt(), din.readInt(), @@ -137,17 +139,30 @@ public interface Matter { Iris.addPanic("read.matter.header", matter.getHeader().toString()); for (int i = 0; i < sliceCount; i++) { + long size = din.readInt(); + if (size == 0) continue; + long start = din.count(); + Iris.addPanic("read.matter.slice", i + ""); - String cn = din.readUTF(); - Iris.addPanic("read.matter.slice.class", cn); try { + String cn = din.readUTF(); + Iris.addPanic("read.matter.slice.class", cn); + Class type = Class.forName(cn); MatterSlice slice = matter.createSlice(type, matter); slice.read(din); matter.putSlice(type, slice); } catch (Throwable e) { + long end = start + size; + Iris.error("Failed to read matter slice, skipping it."); + Iris.addPanic("read.byte.range", start + " " + end); + Iris.addPanic("read.byte.current", din.count() + ""); + Iris.reportError(e); e.printStackTrace(); - throw new IOException("Can't read class '" + cn + "' (slice count reverse at " + sliceCount + ")"); + Iris.panic(); + + din.skipTo(end); + TectonicPlate.addError(); } } @@ -414,8 +429,16 @@ public interface Matter { dos.writeByte(getSliceTypes().size()); getHeader().write(dos); + var bytes = new ByteArrayOutputStream(1024); + var sub = new DataOutputStream(bytes); for (Class i : getSliceTypes()) { - getSlice(i).write(dos); + try { + getSlice(i).write(sub); + dos.writeInt(bytes.size()); + bytes.writeTo(dos); + } finally { + bytes.reset(); + } } }