Merge pull request #3 from CrazyDev05/Pixeldev

TectonicPlate unloading improvements

core/src/main/java/com/volmit/iris/core/commands/CommandDeveloper.java

@@ -21,6 +21,7 @@ package com.volmit.iris.core.commands;
 import com.volmit.iris.Iris;
 import com.volmit.iris.core.IrisSettings;
 import com.volmit.iris.core.loader.IrisData;
+import com.volmit.iris.core.service.IrisEngineSVC;
 import com.volmit.iris.core.tools.IrisToolbelt;
 import com.volmit.iris.engine.framework.Engine;
 import com.volmit.iris.engine.mantle.EngineMantle;
@@ -56,7 +57,7 @@ public class CommandDeveloper implements DecreeExecutor {
             Iris.info("-------------------------");
             Iris.info(C.DARK_PURPLE + "Engine Status");
             Iris.info(C.DARK_PURPLE + "Tectonic Threads: " + C.LIGHT_PURPLE + engine.getMantle().getDynamicThreads());
-            Iris.info(C.DARK_PURPLE + "Tectonic Limit: " + C.LIGHT_PURPLE + engine.getMantle().getTectonicLimit());
+            Iris.info(C.DARK_PURPLE + "Tectonic Limit: " + C.LIGHT_PURPLE + IrisEngineSVC.getTectonicLimit());
             Iris.info(C.DARK_PURPLE + "Tectonic Loaded Plates: " + C.LIGHT_PURPLE + engine.getMantle().getLoadedRegionCount());
             Iris.info(C.DARK_PURPLE + "Tectonic Plates: " + C.LIGHT_PURPLE + engine.getMantle().getNotClearedLoadedRegions());
             Iris.info(C.DARK_PURPLE + "Tectonic ToUnload: " + C.LIGHT_PURPLE + engine.getMantle().getToUnload());

core/src/main/java/com/volmit/iris/core/loader/ResourceLoader.java

@@ -40,6 +40,8 @@ import com.volmit.iris.util.scheduling.ChronoLatch;
 import com.volmit.iris.util.scheduling.J;
 import com.volmit.iris.util.scheduling.PrecisionStopwatch;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
 
 import java.io.*;
 import java.util.Locale;
@@ -52,6 +54,8 @@ import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 
 @Data
+@EqualsAndHashCode(exclude = "manager")
+@ToString(exclude = "manager")
 public class ResourceLoader<T extends IrisRegistrant> implements MeteredCache {
     public static final AtomicDouble tlt = new AtomicDouble(0);
     private static final int CACHE_SIZE = 100000;

core/src/main/java/com/volmit/iris/core/pregenerator/IrisPregenerator.java

@@ -167,7 +167,7 @@ public class IrisPregenerator {
         generator.close();
         ticker.interrupt();
         listener.onClose();
-        getMantle().trim(0);
+        getMantle().trim(0, 0);
     }
 
     private void visitRegion(int x, int z, boolean regions) {

core/src/main/java/com/volmit/iris/core/service/IrisEngineSVC.java

@@ -4,30 +4,26 @@ import com.volmit.iris.Iris;
 import com.volmit.iris.core.tools.IrisToolbelt;
 import com.volmit.iris.engine.framework.Engine;
 import com.volmit.iris.engine.platform.PlatformChunkGenerator;
+import com.volmit.iris.util.collection.KMap;
+import com.volmit.iris.util.format.C;
+import com.volmit.iris.util.format.Form;
 import com.volmit.iris.util.misc.getHardware;
 import com.volmit.iris.util.plugin.IrisService;
-import com.volmit.iris.util.scheduling.J;
 import com.volmit.iris.util.scheduling.Looper;
 import org.bukkit.Bukkit;
 import org.bukkit.World;
-import org.bukkit.plugin.java.JavaPlugin;
 
 import java.util.ArrayList;
-import java.util.Iterator;
 import java.util.List;
-import java.util.Random;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
 
-import static com.volmit.iris.util.mantle.Mantle.tectonicLimit;
-
 public class IrisEngineSVC implements IrisService {
-    public Looper trimTicker;
+    private static final AtomicInteger tectonicLimit = new AtomicInteger(30);
-    public Looper unloadTicker;
+    private final KMap<Engine, Long> cache = new KMap<>();
+    private Looper cacheTicker;
+    private Looper trimTicker;
+    private Looper unloadTicker;
     public List<World> corruptedIrisWorlds = new ArrayList<>();
 
     // todo make this work with multiple worlds
@@ -42,49 +38,79 @@ public class IrisEngineSVC implements IrisService {
         }
         tectonicLimit.set(10); // DEBUG CODE
         this.setup();
+        cacheTicker.start();
         trimTicker.start();
         unloadTicker.start();
     }
 
-    private void setup() {
+    public static int getTectonicLimit() {
-        trimTicker = new Looper() {
+        return tectonicLimit.get();
-            private final Supplier<Engine> supplier = createSupplier();
+    }
-            private Engine engine = supplier.get();
 
+    private void setup() {
+        cacheTicker = new Looper() {
             @Override
             protected long loop() {
-                try {
+                long now = System.currentTimeMillis();
-                    if (engine != null) {
+                for (Engine key : cache.keySet()) {
-                        engine.getMantle().trim();
+                    Long last = cache.get(key);
+                    if (last == null)
+                        continue;
+                    if (now - last > 600000) { // 10 minutes
+                        cache.remove(key);
+                    }
+                }
+                return 1000;
+            }
+        };
+        trimTicker = new Looper() {
+            private final Supplier<Engine> supplier = createSupplier();
+            @Override
+            protected long loop() {
+                long start = System.currentTimeMillis();
+                try {
+                    Engine engine = supplier.get();
+                    if (engine != null) {
+                        engine.getMantle().trim(tectonicLimit.get() / cache.size());
                     }
-                    engine = supplier.get();
                 } catch (Throwable e) {
                     Iris.reportError(e);
-                    e.printStackTrace();
                     return -1;
                 }
 
-                return 1000;
+                int size = cache.size();
+                long time = (size > 0 ? 1000/size : 1000) - (System.currentTimeMillis() - start);
+                if (time <= 0)
+                    return 0;
+                return time;
             }
         };
 
         unloadTicker = new Looper() {
             private final Supplier<Engine> supplier = createSupplier();
-            private Engine engine = supplier.get();
 
             @Override
             protected long loop() {
+                long start = System.currentTimeMillis();
                 try {
+                    Engine engine = supplier.get();
                     if (engine != null) {
-                        engine.getMantle().unloadTectonicPlate();
+                        long unloadStart = System.currentTimeMillis();
+                        int count = engine.getMantle().unloadTectonicPlate();
+                        if (count > 0) {
+                            Iris.info("Unloaded " + count + " TectonicPlates in " + C.RED + Form.duration(System.currentTimeMillis() - unloadStart, 2));
+                        }
                     }
-                    engine = supplier.get();
                 } catch (Throwable e) {
                     Iris.reportError(e);
-                    e.printStackTrace();
                     return -1;
                 }
-                return 1000;
+
+                int size = cache.size();
+                long time = (size > 0 ? 1000/size : 1000) - (System.currentTimeMillis() - start);
+                if (time <= 0)
+                    return 0;
+                return time;
             }
         };
     }
@@ -96,15 +122,23 @@ public class IrisEngineSVC implements IrisService {
             if (i.get() >= worlds.size()) {
                 i.set(0);
             }
-            for (int j = 0; j < worlds.size(); j++) {
+            try {
-                PlatformChunkGenerator generator = IrisToolbelt.access(worlds.get(i.getAndIncrement()));
+                for (int j = 0; j < worlds.size(); j++) {
-                if (i.get() >= worlds.size()) {
+                    PlatformChunkGenerator generator = IrisToolbelt.access(worlds.get(i.getAndIncrement()));
-                    i.set(0);
+                    if (i.get() >= worlds.size()) {
-                }
+                        i.set(0);
+                    }
 
-                if (generator != null && generator.getEngine() != null) {
+                    if (generator != null) {
-                    return generator.getEngine();
+                        Engine engine = generator.getEngine();
+                        if (engine != null) {
+                            cache.put(engine, System.currentTimeMillis());
+                            return engine;
+                        }
+                    }
                 }
+            } catch (Throwable e) {
+                Iris.reportError(e);
             }
             return null;
         };
@@ -112,7 +146,9 @@ public class IrisEngineSVC implements IrisService {
 
     @Override
     public void onDisable() {
+        cacheTicker.interrupt();
         trimTicker.interrupt();
         unloadTicker.interrupt();
+        cache.clear();
     }
 }

core/src/main/java/com/volmit/iris/engine/IrisComplex.java

@@ -35,6 +35,8 @@ import com.volmit.iris.util.noise.CNG;
 import com.volmit.iris.util.stream.ProceduralStream;
 import com.volmit.iris.util.stream.interpolation.Interpolated;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
 import org.bukkit.Material;
 import org.bukkit.block.Biome;
 import org.bukkit.block.data.BlockData;
@@ -42,6 +44,8 @@ import org.bukkit.block.data.BlockData;
 import java.util.UUID;
 
 @Data
+@EqualsAndHashCode(exclude = "data")
+@ToString(exclude = "data")
 public class IrisComplex implements DataProvider {
     private static final BlockData AIR = Material.AIR.createBlockData();
     private RNG rng;

core/src/main/java/com/volmit/iris/engine/IrisEngine.java

@@ -50,6 +50,8 @@ import com.volmit.iris.util.scheduling.ChronoLatch;
 import com.volmit.iris.util.scheduling.J;
 import com.volmit.iris.util.scheduling.PrecisionStopwatch;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
 import org.bukkit.Material;
 import org.bukkit.block.Biome;
 import org.bukkit.block.data.BlockData;
@@ -64,6 +66,8 @@ import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
 @Data
+@EqualsAndHashCode(exclude = "context")
+@ToString(exclude = "context")
 public class IrisEngine implements Engine {
     private final AtomicInteger bud;
     private final AtomicInteger buds;

core/src/main/java/com/volmit/iris/engine/IrisEngineMantle.java

@@ -38,6 +38,8 @@ import com.volmit.iris.util.format.Form;
 import com.volmit.iris.util.mantle.Mantle;
 import com.volmit.iris.util.parallel.BurstExecutor;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
 import org.bukkit.util.BlockVector;
 
 import java.io.File;
@@ -49,6 +51,8 @@ import static com.volmit.iris.core.tools.IrisPackBenchmarking.benchmark;
 import static com.volmit.iris.core.safeguard.PerformanceSFG.*;
 
 @Data
+@EqualsAndHashCode(exclude = "engine")
+@ToString(exclude = "engine")
 public class IrisEngineMantle implements EngineMantle {
     private final Engine engine;
     private final Mantle mantle;

core/src/main/java/com/volmit/iris/engine/IrisExecutionEnvironment.java

@@ -25,11 +25,15 @@ import com.volmit.iris.engine.scripting.EngineExecutionEnvironment;
 import com.volmit.iris.engine.scripting.IrisScriptingAPI;
 import com.volmit.iris.util.format.C;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
 import org.apache.bsf.BSFException;
 import org.apache.bsf.BSFManager;
 import org.apache.bsf.engines.javascript.JavaScriptEngine;
 
 @Data
+@EqualsAndHashCode(exclude = "engine")
+@ToString(exclude = "engine")
 public class IrisExecutionEnvironment implements EngineExecutionEnvironment {
     private final BSFManager manager;
     private final Engine engine;

core/src/main/java/com/volmit/iris/engine/framework/Engine.java

@@ -523,8 +523,9 @@ public interface Engine extends DataProvider, Fallible, LootProvider, BlockUpdat
         return getTarget().getBurster();
     }
 
+    @Deprecated
     default void clean() {
-        burst().lazy(() -> getMantle().trim());
+        burst().lazy(() -> getMantle().trim(10));
     }
 
     @BlockCoordinates

core/src/main/java/com/volmit/iris/engine/framework/EngineAssignedComponent.java

@@ -21,8 +21,12 @@ package com.volmit.iris.engine.framework;
 import com.volmit.iris.Iris;
 import com.volmit.iris.util.math.RollingSequence;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
 
 @Data
+@EqualsAndHashCode(exclude = "engine")
+@ToString(exclude = "engine")
 public class EngineAssignedComponent implements EngineComponent {
     private final Engine engine;
     private final RollingSequence metrics;

core/src/main/java/com/volmit/iris/engine/framework/EngineTarget.java

@@ -23,8 +23,12 @@ import com.volmit.iris.engine.object.IrisDimension;
 import com.volmit.iris.engine.object.IrisWorld;
 import com.volmit.iris.util.parallel.MultiBurst;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
 
 @Data
+@EqualsAndHashCode(exclude = "data")
+@ToString(exclude = "data")
 public class EngineTarget {
     private final MultiBurst burster;
     private final IrisData data;

core/src/main/java/com/volmit/iris/engine/mantle/EngineMantle.java

@@ -143,8 +143,8 @@ public interface EngineMantle extends IObjectPlacer {
         return getEngine().getDimension().isDebugSmartBore();
     }
 
-    default void trim(long dur) {
+    default void trim(long dur, int limit) {
-        getMantle().trim(dur);
+        getMantle().trim(dur, limit);
     }
 
     default IrisData getData() {
@@ -175,11 +175,11 @@ public interface EngineMantle extends IObjectPlacer {
 
     }
 
-    default void trim() {
+    default void trim(int limit) {
-        getMantle().trim(TimeUnit.SECONDS.toMillis(IrisSettings.get().getPerformance().getMantleKeepAlive()));
+        getMantle().trim(TimeUnit.SECONDS.toMillis(IrisSettings.get().getPerformance().getMantleKeepAlive()), limit);
     }
-    default void unloadTectonicPlate(){
+    default int unloadTectonicPlate(){
-        getMantle().unloadTectonicPlate();
+        return getMantle().unloadTectonicPlate();
     }
 
     default MultiBurst burst() {
@@ -301,9 +301,6 @@ public interface EngineMantle extends IObjectPlacer {
     default long getDynamicThreads(){
         return getMantle().getDynamicThreads().get();
     }
-    default double getTectonicLimit(){
-        return Mantle.tectonicLimit.get();
-    }
     default long getNotClearedLoadedRegions(){
         return getMantle().getLoadedRegions().size() - getMantle().getToUnload().size();
     }

core/src/main/java/com/volmit/iris/engine/mantle/IrisMantleComponent.java

@@ -20,8 +20,12 @@ package com.volmit.iris.engine.mantle;
 
 import com.volmit.iris.util.mantle.MantleFlag;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
 
 @Data
+@EqualsAndHashCode(exclude = "engineMantle")
+@ToString(exclude = "engineMantle")
 public abstract class IrisMantleComponent implements MantleComponent {
     private final EngineMantle engineMantle;
     private final MantleFlag flag;

core/src/main/java/com/volmit/iris/engine/scripting/IrisScriptingAPI.java

@@ -27,10 +27,14 @@ import com.volmit.iris.engine.object.IrisBiome;
 import com.volmit.iris.engine.object.IrisDimension;
 import com.volmit.iris.engine.object.IrisExpression;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
 import org.bukkit.Location;
 import org.bukkit.entity.Entity;
 
 @Data
+@EqualsAndHashCode(exclude = "engine")
+@ToString(exclude = "engine")
 public class IrisScriptingAPI {
     private final Engine engine;
     private IrisRegistrant preprocessorObject;

core/src/main/java/com/volmit/iris/util/mantle/Mantle.java

@@ -392,7 +392,6 @@ public class Mantle {
 
     @Getter
     private final AtomicDouble adjustedIdleDuration = new AtomicDouble(0);
-    public static final AtomicInteger tectonicLimit = new AtomicInteger(30);
     @Getter
     private final AtomicInteger dynamicThreads = new AtomicInteger(4);
     @Getter
@@ -400,8 +399,7 @@ public class Mantle {
     @Getter
     private final AtomicLong oldestTectonicPlate = new AtomicLong(0);
     @Getter
-    public final Set<Long> toUnload = new HashSet<>();
+    private Set<Long> toUnload = new HashSet<>();
-    private int g = 0;
 
     /**
      * Save & unload regions that have not been used for more than the
@@ -409,7 +407,7 @@ public class Mantle {
      *
      * @param baseIdleDuration the duration
      */
-    public synchronized void trim(long baseIdleDuration) {
+    public synchronized void trim(long baseIdleDuration, int tectonicLimit) {
         if (closed.get()) {
             throw new RuntimeException("The Mantle is closed");
         }
@@ -418,10 +416,9 @@ public class Mantle {
         adjustedIdleDuration.set(baseIdleDuration);
 
         if (loadedRegions != null) {
-            if (loadedRegions.size() > tectonicLimit.get()) {
+            if (loadedRegions.size() > tectonicLimit) {
                 // todo update this correctly and maybe do something when its above a 100%
-                int tectonicLimitValue = tectonicLimit.get();
+                adjustedIdleDuration.set(Math.max(adjustedIdleDuration.get() - (1000 * (((loadedRegions.size() - tectonicLimit) / (double) tectonicLimit) * 100) * 0.4), 4000));
-                adjustedIdleDuration.set(Math.max(adjustedIdleDuration.get() - (1000 * (((loadedRegions.size() - tectonicLimitValue) / (double) tectonicLimitValue) * 100) * 0.4), 4000));
             }
         }
 
@@ -446,28 +443,42 @@ public class Mantle {
         }
     }
 
-    public void unloadTectonicPlate() {
+    public int unloadTectonicPlate() {
+        AtomicInteger i = new AtomicInteger();
+        Set<Long> toUnload = this.toUnload;
+        this.toUnload = new HashSet<>();
         try {
+            List<Future<?>> futures = new ArrayList<>();
+            ExecutorService service = Executors.newFixedThreadPool(dynamicThreads.get());
             for (Long id : new ArrayList<>(toUnload)) {
-                hyperLock.withLong(id, () -> {
+                futures.add(service.submit(() ->
-                    TectonicPlate m = loadedRegions.get(id);
+                        hyperLock.withLong(id, () -> {
-                    if (m != null) {
+                            TectonicPlate m = loadedRegions.get(id);
-                        try {
+                            if (m != null) {
-                            m.write(fileForRegion(dataFolder, id));
+                                try {
-                            loadedRegions.remove(id);
+                                    m.write(fileForRegion(dataFolder, id));
-                            lastUse.remove(id);
+                                    loadedRegions.remove(id);
-                            toUnload.remove(id);
+                                    lastUse.remove(id);
-                            Iris.info("Unloaded Tectonic Plate " + C.DARK_GREEN + Cache.keyX(id) + " " + Cache.keyZ(id));
+                                    toUnload.remove(id);
-                        } catch (IOException e) {
+                                    i.incrementAndGet();
-                            e.printStackTrace();
+                                    Iris.info("Unloaded Tectonic Plate " + C.DARK_GREEN + Cache.keyX(id) + " " + Cache.keyZ(id));
-                        }
+                                } catch (IOException e) {
-                    }
+                                    e.printStackTrace();
-                });
+                                }
+                            }
+                        })));
             }
+            while (!futures.isEmpty()) {
+                futures.remove(0).get();
+            }
+            service.shutdown();
         } catch (Exception e) {
             e.printStackTrace();
+        } finally {
+            this.toUnload.addAll(toUnload);
         }
         ioTectonicUnload.set(true);
+        return i.get();
     }