fix deadlock when hotloading / closing engine

core/src/main/java/com/volmit/iris/core/commands/CommandDeveloper.java

@@ -22,11 +22,11 @@ import com.volmit.iris.Iris;
 import com.volmit.iris.core.ServerConfigurator;
 import com.volmit.iris.core.loader.IrisData;
 import com.volmit.iris.core.nms.datapack.DataVersion;
-import com.volmit.iris.core.service.IrisEngineSVC;
 import com.volmit.iris.core.tools.IrisPackBenchmarking;
 import com.volmit.iris.core.tools.IrisToolbelt;
 import com.volmit.iris.engine.framework.Engine;
 import com.volmit.iris.engine.object.IrisDimension;
+import com.volmit.iris.engine.service.EngineStatusSVC;
 import com.volmit.iris.util.decree.DecreeExecutor;
 import com.volmit.iris.util.decree.DecreeOrigin;
 import com.volmit.iris.util.decree.annotations.Decree;
@@ -66,53 +66,20 @@ public class CommandDeveloper implements DecreeExecutor {
 
     @Decree(description = "Get Loaded TectonicPlates Count", origin = DecreeOrigin.BOTH, sync = true)
     public void EngineStatus() {
-        List<World> IrisWorlds = new ArrayList<>();
+        var status = EngineStatusSVC.getStatus();
-        int TotalLoadedChunks = 0;
-        int TotalQueuedTectonicPlates = 0;
-        int TotalNotQueuedTectonicPlates = 0;
-        int TotalTectonicPlates = 0;
 
-        long lowestUnloadDuration = 0;
+        sender().sendMessage("-------------------------");
-        long highestUnloadDuration = 0;
+        sender().sendMessage(C.DARK_PURPLE + "Engine Status");
-
+        sender().sendMessage(C.DARK_PURPLE + "Total Engines: " + C.LIGHT_PURPLE + status.engineCount());
-        for (World world : Bukkit.getWorlds()) {
+        sender().sendMessage(C.DARK_PURPLE + "Total Loaded Chunks: " + C.LIGHT_PURPLE + status.loadedChunks());
-            try {
+        sender().sendMessage(C.DARK_PURPLE + "Tectonic Limit: " + C.LIGHT_PURPLE + status.tectonicLimit());
-                if (IrisToolbelt.access(world).getEngine() != null) {
+        sender().sendMessage(C.DARK_PURPLE + "Tectonic Total Plates: " + C.LIGHT_PURPLE + status.tectonicPlates());
-                    IrisWorlds.add(world);
+        sender().sendMessage(C.DARK_PURPLE + "Tectonic Active Plates: " + C.LIGHT_PURPLE + status.activeTectonicPlates());
-                }
+        sender().sendMessage(C.DARK_PURPLE + "Tectonic ToUnload: " + C.LIGHT_PURPLE + status.queuedTectonicPlates());
-            } catch (Exception e) {
+        sender().sendMessage(C.DARK_PURPLE + "Lowest Tectonic Unload Duration: " + C.LIGHT_PURPLE + Form.duration(status.minTectonicUnloadDuration()));
-                // no
+        sender().sendMessage(C.DARK_PURPLE + "Highest Tectonic Unload Duration: " + C.LIGHT_PURPLE + Form.duration(status.maxTectonicUnloadDuration()));
-            }
+        sender().sendMessage(C.DARK_PURPLE + "Cache Size: " + C.LIGHT_PURPLE + Form.f(IrisData.cacheSize()));
-        }
+        sender().sendMessage("-------------------------");
-
-        for (World world : IrisWorlds) {
-            Engine engine = IrisToolbelt.access(world).getEngine();
-            TotalQueuedTectonicPlates += (int) engine.getMantle().getToUnload();
-            TotalNotQueuedTectonicPlates += (int) engine.getMantle().getNotQueuedLoadedRegions();
-            TotalTectonicPlates += engine.getMantle().getLoadedRegionCount();
-            if (highestUnloadDuration <= (long) engine.getMantle().getTectonicDuration()) {
-                highestUnloadDuration = (long) engine.getMantle().getTectonicDuration();
-            }
-            if (lowestUnloadDuration >= (long) engine.getMantle().getTectonicDuration()) {
-                lowestUnloadDuration = (long) engine.getMantle().getTectonicDuration();
-            }
-            for (Chunk chunk : world.getLoadedChunks()) {
-                if (chunk.isLoaded()) {
-                    TotalLoadedChunks++;
-                }
-            }
-        }
-        Iris.info("-------------------------");
-        Iris.info(C.DARK_PURPLE + "Engine Status");
-        Iris.info(C.DARK_PURPLE + "Total Loaded Chunks: " + C.LIGHT_PURPLE + TotalLoadedChunks);
-        Iris.info(C.DARK_PURPLE + "Tectonic Limit: " + C.LIGHT_PURPLE + IrisEngineSVC.getTectonicLimit());
-        Iris.info(C.DARK_PURPLE + "Tectonic Total Plates: " + C.LIGHT_PURPLE + TotalTectonicPlates);
-        Iris.info(C.DARK_PURPLE + "Tectonic Active Plates: " + C.LIGHT_PURPLE + TotalNotQueuedTectonicPlates);
-        Iris.info(C.DARK_PURPLE + "Tectonic ToUnload: " + C.LIGHT_PURPLE + TotalQueuedTectonicPlates);
-        Iris.info(C.DARK_PURPLE + "Lowest Tectonic Unload Duration: " + C.LIGHT_PURPLE + Form.duration(lowestUnloadDuration));
-        Iris.info(C.DARK_PURPLE + "Highest Tectonic Unload Duration: " + C.LIGHT_PURPLE + Form.duration(highestUnloadDuration));
-        Iris.info(C.DARK_PURPLE + "Cache Size: " + C.LIGHT_PURPLE + Form.f(IrisData.cacheSize()));
-        Iris.info("-------------------------");
     }
 
     @Decree(description = "Test")

core/src/main/java/com/volmit/iris/core/commands/CommandSupport.java

@@ -21,7 +21,6 @@ package com.volmit.iris.core.commands;
 import com.volmit.iris.Iris;
 import com.volmit.iris.core.loader.IrisData;
 import com.volmit.iris.core.pregenerator.ChunkUpdater;
-import com.volmit.iris.core.service.IrisEngineSVC;
 import com.volmit.iris.core.tools.IrisPackBenchmarking;
 import com.volmit.iris.core.tools.IrisToolbelt;
 import com.volmit.iris.engine.framework.Engine;

core/src/main/java/com/volmit/iris/core/loader/ResourceLoader.java

@@ -44,6 +44,7 @@ import lombok.EqualsAndHashCode;
 import lombok.ToString;
 
 import java.io.*;
+import java.util.Arrays;
 import java.util.Locale;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
@@ -219,6 +220,10 @@ public class ResourceLoader<T extends IrisRegistrant> implements MeteredCache {
         return s.map(this::load);
     }
 
+    public Stream<T> streamAllPossible() {
+        return streamAll(Arrays.stream(getPossibleKeys()));
+    }
+
     public KList<T> loadAll(KList<String> s) {
         KList<T> m = new KList<>();
 

core/src/main/java/com/volmit/iris/core/service/IrisEngineSVC.java

@@ -1,317 +0,0 @@
-package com.volmit.iris.core.service;
-
-import com.volmit.iris.Iris;
-import com.volmit.iris.core.IrisSettings;
-import com.volmit.iris.core.tools.IrisToolbelt;
-import com.volmit.iris.engine.framework.Engine;
-import com.volmit.iris.engine.platform.PlatformChunkGenerator;
-import com.volmit.iris.util.collection.KList;
-import com.volmit.iris.util.collection.KMap;
-import com.volmit.iris.util.format.C;
-import com.volmit.iris.util.format.Form;
-import com.volmit.iris.util.mantle.TectonicPlate;
-import com.volmit.iris.util.misc.getHardware;
-import com.volmit.iris.util.plugin.IrisService;
-import com.volmit.iris.util.scheduling.ChronoLatch;
-import com.volmit.iris.util.scheduling.Looper;
-import com.volmit.iris.util.scheduling.PrecisionStopwatch;
-import org.bukkit.Bukkit;
-import org.bukkit.World;
-import org.bukkit.event.EventHandler;
-import org.bukkit.event.server.PluginDisableEvent;
-import org.bukkit.event.server.ServerLoadEvent;
-import org.bukkit.event.world.WorldLoadEvent;
-import org.bukkit.event.world.WorldUnloadEvent;
-import org.checkerframework.checker.units.qual.A;
-
-import java.util.*;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.function.Supplier;
-
-public class IrisEngineSVC implements IrisService {
-    public static IrisEngineSVC instance;
-    public boolean isServerShuttingDown = false;
-    public boolean isServerLoaded = false;
-    private static final AtomicInteger tectonicLimit = new AtomicInteger(30);
-    private ReentrantLock lastUseLock;
-    private KMap<World, Long> lastUse;
-    private List<World> IrisWorlds;
-    private Looper cacheTicker;
-    private Looper trimTicker;
-    private Looper unloadTicker;
-    private Looper updateTicker;
-    private PrecisionStopwatch trimAlive;
-    private PrecisionStopwatch unloadAlive;
-    public PrecisionStopwatch trimActiveAlive;
-    public PrecisionStopwatch unloadActiveAlive;
-    private AtomicInteger TotalTectonicPlates;
-    private AtomicInteger TotalQueuedTectonicPlates;
-    private AtomicInteger TotalNotQueuedTectonicPlates;
-    private AtomicBoolean IsUnloadAlive;
-    private AtomicBoolean IsTrimAlive;
-    ChronoLatch cl;
-
-    public List<World> corruptedIrisWorlds = new ArrayList<>();
-
-    @Override
-    public void onEnable() {
-        this.cl = new ChronoLatch(5000);
-        lastUse = new KMap<>();
-        lastUseLock = new ReentrantLock();
-        IrisWorlds = new ArrayList<>();
-        IsUnloadAlive = new AtomicBoolean(true);
-        IsTrimAlive = new AtomicBoolean(true);
-        trimActiveAlive = new PrecisionStopwatch();
-        unloadActiveAlive = new PrecisionStopwatch();
-        trimAlive = new PrecisionStopwatch();
-        unloadAlive = new PrecisionStopwatch();
-        TotalTectonicPlates = new AtomicInteger();
-        TotalQueuedTectonicPlates = new AtomicInteger();
-        TotalNotQueuedTectonicPlates = new AtomicInteger();
-        tectonicLimit.set(2);
-        long t = getHardware.getProcessMemory();
-        while (t > 200) {
-            tectonicLimit.getAndAdd(1);
-            t = t - 200;
-        }
-        this.setup();
-        this.TrimLogic();
-        this.UnloadLogic();
-
-        trimAlive.begin();
-        unloadAlive.begin();
-        trimActiveAlive.begin();
-        unloadActiveAlive.begin();
-
-        updateTicker.start();
-        cacheTicker.start();
-        //trimTicker.start();
-        //unloadTicker.start();
-        instance = this;
-
-    }
-
-    public void engineStatus() {
-        boolean trimAlive = trimTicker.isAlive();
-        boolean unloadAlive = unloadTicker.isAlive();
-        Iris.info("Status:");
-        Iris.info("- Trim: " + trimAlive);
-        Iris.info("- Unload: " + unloadAlive);
-
-    }
-
-    public static int getTectonicLimit() {
-        return tectonicLimit.get();
-    }
-
-    @EventHandler
-    public void onWorldUnload(WorldUnloadEvent event) {
-        updateWorlds();
-    }
-
-    @EventHandler
-    public void onWorldLoad(WorldLoadEvent event) {
-        updateWorlds();
-    }
-
-    @EventHandler
-    public void onServerBoot(ServerLoadEvent event) {
-        isServerLoaded = true;
-    }
-
-    @EventHandler
-    public void onPluginDisable(PluginDisableEvent event) {
-        if (event.getPlugin().equals(Iris.instance)) {
-            isServerShuttingDown = true;
-        }
-    }
-
-    public void updateWorlds() {
-        for (World world : Bukkit.getWorlds()) {
-            try {
-                if (IrisToolbelt.access(world).getEngine() != null) {
-                    IrisWorlds.add(world);
-                }
-            } catch (Exception e) {
-                // no
-            }
-        }
-    }
-
-    private void setup() {
-        cacheTicker = new Looper() {
-            @Override
-            protected long loop() {
-                long now = System.currentTimeMillis();
-                lastUseLock.lock();
-                try {
-                    for (World key : new ArrayList<>(lastUse.keySet())) {
-                        Long last = lastUse.get(key);
-                        if (last == null)
-                            continue;
-                        if (now - last > 60000) {
-                            lastUse.remove(key);
-                        }
-                    }
-                } finally {
-                    lastUseLock.unlock();
-                }
-                return 1000;
-            }
-        };
-
-        updateTicker = new Looper() {
-            @Override
-            protected long loop() {
-                try {
-                    TotalQueuedTectonicPlates.set(0);
-                    TotalNotQueuedTectonicPlates.set(0);
-                    TotalTectonicPlates.set(0);
-                    for (World world : IrisWorlds) {
-                        Engine engine = Objects.requireNonNull(IrisToolbelt.access(world)).getEngine();
-                        TotalQueuedTectonicPlates.addAndGet((int) engine.getMantle().getToUnload());
-                        TotalNotQueuedTectonicPlates.addAndGet((int) engine.getMantle().getNotQueuedLoadedRegions());
-                        TotalTectonicPlates.addAndGet(engine.getMantle().getLoadedRegionCount());
-                    }
-                    if (!isServerShuttingDown && isServerLoaded) {
-                        if (!trimTicker.isAlive()) {
-                            Iris.info(C.RED + "TrimTicker found dead! Booting it up!");
-                            try {
-                                TrimLogic();
-                            } catch (Exception e) {
-                                Iris.error("What happened?");
-                                e.printStackTrace();
-                            }
-                        }
-
-                        if (!unloadTicker.isAlive()) {
-                            Iris.info(C.RED + "UnloadTicker found dead! Booting it up!");
-                            try {
-                               UnloadLogic();
-                            } catch (Exception e) {
-                                Iris.error("What happened?");
-                                e.printStackTrace();
-                            }
-                        }
-                    }
-
-                } catch (Exception e) {
-                    return -1;
-                }
-                return 1000;
-            }
-        };
-    }
-    public void TrimLogic() {
-        if (trimTicker == null || !trimTicker.isAlive()) {
-            trimTicker = new Looper() {
-                private final Supplier<Engine> supplier = createSupplier();
-
-                @Override
-                protected long loop() {
-                    long start = System.currentTimeMillis();
-                    trimAlive.reset();
-                    try {
-                        Engine engine = supplier.get();
-                        if (engine != null) {
-                            engine.getMantle().trim(tectonicLimit.get() / lastUse.size());
-                        }
-                    } catch (Throwable e) {
-                        Iris.reportError(e);
-                        Iris.info(C.RED + "EngineSVC: Failed to trim.");
-                        e.printStackTrace();
-                        return -1;
-                    }
-
-                    int size = lastUse.size();
-                    long time = (size > 0 ? 1000 / size : 1000) - (System.currentTimeMillis() - start);
-                    if (time <= 0)
-                        return 0;
-                    return time;
-                }
-            };
-            trimTicker.start();
-        }
-    }
-    public void UnloadLogic() {
-        if (unloadTicker == null || !unloadTicker.isAlive()) {
-            unloadTicker = new Looper() {
-                private final Supplier<Engine> supplier = createSupplier();
-
-                @Override
-                protected long loop() {
-                    long start = System.currentTimeMillis();
-                    unloadAlive.reset();
-                    try {
-                        Engine engine = supplier.get();
-                        if (engine != null) {
-                            long unloadStart = System.currentTimeMillis();
-                            int count = engine.getMantle().unloadTectonicPlate(tectonicLimit.get() / lastUse.size());
-                            if (count > 0) {
-                                Iris.debug(C.GOLD + "Unloaded " + C.YELLOW + count + " TectonicPlates in " + C.RED + Form.duration(System.currentTimeMillis() - unloadStart, 2));
-                            }
-                        }
-                    } catch (Throwable e) {
-                        Iris.reportError(e);
-                        Iris.info(C.RED + "EngineSVC: Failed to unload.");
-                        e.printStackTrace();
-                        return -1;
-                    }
-
-                    int size = lastUse.size();
-                    long time = (size > 0 ? 1000 / size : 1000) - (System.currentTimeMillis() - start);
-                    if (time <= 0)
-                        return 0;
-                    return time;
-                }
-            };
-            unloadTicker.start();
-        }
-    }
-
-    private Supplier<Engine> createSupplier() {
-        AtomicInteger i = new AtomicInteger();
-        return () -> {
-            List<World> worlds = Bukkit.getWorlds();
-            if (i.get() >= worlds.size()) {
-                i.set(0);
-            }
-            try {
-                for (int j = 0; j < worlds.size(); j++) {
-                    World world = worlds.get(i.getAndIncrement());
-                    PlatformChunkGenerator generator = IrisToolbelt.access(world);
-                    if (i.get() >= worlds.size()) {
-                        i.set(0);
-                    }
-
-                    if (generator != null) {
-                        Engine engine = generator.getEngine();
-                        boolean closed = engine.getMantle().getData().isClosed();
-                        if (engine != null && !engine.isStudio() && !closed) {
-                            lastUseLock.lock();
-                            lastUse.put(world, System.currentTimeMillis());
-                            lastUseLock.unlock();
-                            return engine;
-                        }
-                    }
-                }
-            } catch (Throwable e) {
-                Iris.info(C.RED + "EngineSVC: Failed to create supplier.");
-                e.printStackTrace();
-                Iris.reportError(e);
-            }
-            return null;
-        };
-    }
-
-    @Override
-    public void onDisable() {
-        cacheTicker.interrupt();
-        trimTicker.interrupt();
-        unloadTicker.interrupt();
-        lastUse.clear();
-    }
-}

core/src/main/java/com/volmit/iris/engine/IrisEngine.java

@@ -43,6 +43,7 @@ import com.volmit.iris.util.format.C;
 import com.volmit.iris.util.format.Form;
 import com.volmit.iris.util.hunk.Hunk;
 import com.volmit.iris.util.io.IO;
+import com.volmit.iris.util.io.JarScanner;
 import com.volmit.iris.util.mantle.MantleFlag;
 import com.volmit.iris.util.math.M;
 import com.volmit.iris.util.math.RNG;
@@ -60,7 +61,11 @@ import org.bukkit.command.CommandSender;
 
 import java.io.File;
 import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.Collections;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -71,6 +76,8 @@ import java.util.concurrent.atomic.AtomicLong;
 @EqualsAndHashCode(exclude = "context")
 @ToString(exclude = "context")
 public class IrisEngine implements Engine {
+    private static final Map<Class<? extends IrisEngineService>, Constructor<? extends IrisEngineService>> SERVICES = scanServices();
+    private final KMap<Class<? extends IrisEngineService>, IrisEngineService> services;
     private final AtomicInteger bud;
     private final AtomicInteger buds;
     private final AtomicInteger generated;
@@ -111,6 +118,7 @@ public class IrisEngine implements Engine {
         getEngineData();
         verifySeed();
         this.seedManager = new SeedManager(target.getWorld().getRawWorldSeed());
+        services = new KMap<>();
         bud = new AtomicInteger(0);
         buds = new AtomicInteger(0);
         metrics = new EngineMetrics(32);
@@ -137,6 +145,26 @@ public class IrisEngine implements Engine {
         Iris.debug("Engine Initialized " + getCacheID());
     }
 
+    @SuppressWarnings("unchecked")
+    private static Map<Class<? extends IrisEngineService>, Constructor<? extends IrisEngineService>> scanServices() {
+        JarScanner js = new JarScanner(Iris.instance.getJarFile(), "com.volmit.iris.engine.service");
+        J.attempt(js::scan);
+        KMap<Class<? extends IrisEngineService>, Constructor<? extends IrisEngineService>> map = new KMap<>();
+        js.getClasses()
+                .stream()
+                .filter(IrisEngineService.class::isAssignableFrom)
+                .map(c -> (Class<? extends IrisEngineService>) c)
+                .forEach(c -> {
+                    try {
+                        map.put(c, c.getConstructor(Engine.class));
+                    } catch (NoSuchMethodException e) {
+                        Iris.warn("Failed to load service " + c.getName() + " due to missing constructor");
+                    }
+                });
+
+        return Collections.unmodifiableMap(map);
+    }
+
     private void verifySeed() {
         if (getEngineData().getSeed() != null && getEngineData().getSeed() != target.getWorld().getRawWorldSeed()) {
             target.getWorld().setRawWorldSeed(getEngineData().getSeed());
@@ -161,6 +189,8 @@ public class IrisEngine implements Engine {
         execution.close();
         effects.close();
         mode.close();
+        services.values().forEach(s -> s.onDisable(true));
+        services.values().forEach(Iris.instance::unregisterListener);
 
         J.a(() -> new IrisProject(getData().getDataFolder()).updateWorkspace());
     }
@@ -169,6 +199,24 @@ public class IrisEngine implements Engine {
         try {
             Iris.debug("Setup Engine " + getCacheID());
             cacheId = RNG.r.nextInt();
+            boolean hotload = true;
+            if (services.isEmpty()) {
+                SERVICES.forEach((s, c) -> {
+                    try {
+                        services.put(s, c.newInstance(this));
+                    } catch (InstantiationException | IllegalAccessException |
+                             InvocationTargetException e) {
+                        Iris.error("Failed to create service " + s.getName());
+                        e.printStackTrace();
+                    }
+                });
+                hotload = false;
+            }
+            for (var service : services.values()) {
+                service.onEnable(hotload);
+                Iris.instance.registerListener(service);
+            }
+
             worldManager = new IrisWorldManager(this);
             complex = new IrisComplex(this);
             execution = new IrisExecutionEnvironment(this);
@@ -418,6 +466,7 @@ public class IrisEngine implements Engine {
         PregeneratorJob.shutdownInstance();
         closed = true;
         J.car(art);
+        services.values().forEach(s -> s.onDisable(false));
         getWorldManager().close();
         getTarget().close();
         saveEngineData();

core/src/main/java/com/volmit/iris/engine/IrisWorldManager.java

@@ -27,7 +27,6 @@ import com.volmit.iris.engine.object.*;
 import com.volmit.iris.util.collection.KList;
 import com.volmit.iris.util.collection.KMap;
 import com.volmit.iris.util.collection.KSet;
-import com.volmit.iris.util.format.Form;
 import com.volmit.iris.util.mantle.Mantle;
 import com.volmit.iris.util.mantle.MantleFlag;
 import com.volmit.iris.util.math.M;
@@ -72,7 +71,6 @@ public class IrisWorldManager extends EngineAssignedWorldManager {
     private final Looper looper;
     private final int id;
     private final KList<Runnable> updateQueue = new KList<>();
-    private final ChronoLatch cl;
     private final ChronoLatch clw;
     private final ChronoLatch ecl;
     private final ChronoLatch cln;
@@ -83,12 +81,10 @@ public class IrisWorldManager extends EngineAssignedWorldManager {
     private long charge = 0;
     private int actuallySpawned = 0;
     private int cooldown = 0;
-    private List<Entity> precount = new KList<>();
     private KSet<Position2> injectBiomes = new KSet<>();
 
     public IrisWorldManager() {
         super(null);
-        cl = null;
         ecl = null;
         cln = null;
         clw = null;
@@ -103,7 +99,6 @@ public class IrisWorldManager extends EngineAssignedWorldManager {
         chunkUpdater = new ChronoLatch(3000);
         chunkDiscovery = new ChronoLatch(5000);
         cln = new ChronoLatch(60000);
-        cl = new ChronoLatch(3000);
         ecl = new ChronoLatch(250);
         clw = new ChronoLatch(1000, true);
         id = engine.getCacheID();
@@ -151,27 +146,12 @@ public class IrisWorldManager extends EngineAssignedWorldManager {
                         engine.getEngineData().cleanup(getEngine());
                     }
 
-                    if (precount != null) {
-                        entityCount = 0;
-                        for (Entity i : precount) {
-                            if (i instanceof LivingEntity) {
-                                if (!i.isDead()) {
-                                    entityCount++;
-                                }
-                            }
-                        }
-
-                        precount = null;
-                    }
-
                     if (energy < 650) {
                         if (ecl.flip()) {
                             energy *= 1 + (0.02 * M.clip((1D - getEntitySaturation()), 0D, 1D));
                             fixEnergy();
                         }
                     }
-
-                    onAsyncTick();
                 }
 
                 return IrisSettings.get().getWorld().getAsyncTickIntervalMS();
@@ -214,7 +194,7 @@ public class IrisWorldManager extends EngineAssignedWorldManager {
                             int finalZ = c.getZ() + z;
                             J.a(() -> getMantle().raiseFlag(finalX, finalZ, MantleFlag.INITIAL_SPAWNED_MARKER,
                                     () -> {
-                                        J.a(() -> spawnIn(cx, true), RNG.r.i(5, 200));
+                                        J.a(() -> spawnIn(cx), RNG.r.i(5, 200));
                                         getSpawnersFromMarkers(cx).forEach((blockf, spawners) -> {
                                             if (spawners.isEmpty()) {
                                                 return;
@@ -222,7 +202,7 @@ public class IrisWorldManager extends EngineAssignedWorldManager {
 
                                             IrisPosition block = new IrisPosition(blockf.getX(), blockf.getY() + getEngine().getWorld().minHeight(), blockf.getZ());
                                             IrisSpawner s = new KList<>(spawners).getRandom();
-                                            spawn(block, s, true);
+                                            spawn(block, s);
                                         });
                                     }));
                         }
@@ -232,73 +212,16 @@ public class IrisWorldManager extends EngineAssignedWorldManager {
         }
     }
 
-    private boolean onAsyncTick() {
-        if (getEngine().isClosed()) {
-            return false;
-        }
-
-        actuallySpawned = 0;
-
-        if (energy < 100) {
-            J.sleep(200);
-            return false;
-        }
-
-        if (!getEngine().getWorld().hasRealWorld()) {
-            Iris.debug("Can't spawn. No real world");
-            J.sleep(5000);
-            return false;
-        }
-
-        double epx = getEntitySaturation();
-        if (epx > IrisSettings.get().getWorld().getTargetSpawnEntitiesPerChunk()) {
-            Iris.debug("Can't spawn. The entity per chunk ratio is at " + Form.pc(epx, 2) + " > 100% (total entities " + entityCount + ")");
-            J.sleep(5000);
-            return false;
-        }
-
-        if (cl.flip()) {
-            try {
-                J.s(() -> precount = getEngine().getWorld().realWorld().getEntities());
-            } catch (Throwable e) {
-                close();
-            }
-        }
-
-        int spawnBuffer = RNG.r.i(2, 12);
-
-        Chunk[] cc = getEngine().getWorld().realWorld().getLoadedChunks();
-        while (spawnBuffer-- > 0) {
-            if (cc.length == 0) {
-                Iris.debug("Can't spawn. No chunks!");
-                return false;
-            }
-
-            Chunk c = cc[RNG.r.nextInt(cc.length)];
-
-            if (!c.isLoaded() || !Chunks.isSafe(c.getWorld(), c.getX(), c.getZ())) {
-                continue;
-            }
-
-            spawnIn(c, false);
-        }
-
-        energy -= (actuallySpawned / 2D);
-        return actuallySpawned > 0;
-    }
-
     private void fixEnergy() {
         energy = M.clip(energy, 1D, getDimension().getMaximumEnergy());
     }
 
-    private void spawnIn(Chunk c, boolean initial) {
+    private void spawnIn(Chunk c) {
         if (getEngine().isClosed()) {
             return;
         }
 
-        if (initial) {
+        energy += 1.2;
-            energy += 1.2;
-        }
 
         if (IrisSettings.get().getWorld().isMarkerEntitySpawningSystem()) {
             getSpawnersFromMarkers(c).forEach((blockf, spawners) -> {
@@ -308,9 +231,8 @@ public class IrisWorldManager extends EngineAssignedWorldManager {
 
                 IrisPosition block = new IrisPosition(blockf.getX(), blockf.getY() + getEngine().getWorld().minHeight(), blockf.getZ());
                 IrisSpawner s = new KList<>(spawners).getRandom();
-                spawn(block, s, false);
                 J.a(() -> getMantle().raiseFlag(c.getX(), c.getZ(), MantleFlag.INITIAL_SPAWNED_MARKER,
-                        () -> spawn(block, s, true)));
+                        () -> spawn(block, s)));
             });
         }
 
@@ -341,7 +263,7 @@ public class IrisWorldManager extends EngineAssignedWorldManager {
                                         .stream()
                                         .filter(filter)))
                 .filter(counter)
-                .flatMap((i) -> stream(i, initial))
+                .flatMap(this::stream)
                 .collect(Collectors.toList()))
                 .getRandom();
         //@done
@@ -378,13 +300,13 @@ public class IrisWorldManager extends EngineAssignedWorldManager {
         }
     }
 
-    private Stream<IrisEntitySpawn> stream(IrisSpawner s, boolean initial) {
+    private Stream<IrisEntitySpawn> stream(IrisSpawner s) {
-        for (IrisEntitySpawn i : initial ? s.getInitialSpawns() : s.getSpawns()) {
+        for (IrisEntitySpawn i : s.getInitialSpawns()) {
             i.setReferenceSpawner(s);
             i.setReferenceMarker(s.getReferenceMarker());
         }
 
-        return (initial ? s.getInitialSpawns() : s.getSpawns()).stream();
+        return (s.getInitialSpawns()).stream();
     }
 
     private KList<IrisEntitySpawn> spawnRandomly(List<IrisEntitySpawn> types) {
@@ -431,7 +353,7 @@ public class IrisWorldManager extends EngineAssignedWorldManager {
         }
     }
 
-    private void spawn(IrisPosition block, IrisSpawner spawner, boolean initial) {
+    private void spawn(IrisPosition block, IrisSpawner spawner) {
         if (getEngine().isClosed()) {
             return;
         }
@@ -440,7 +362,7 @@ public class IrisWorldManager extends EngineAssignedWorldManager {
             return;
         }
 
-        KList<IrisEntitySpawn> s = initial ? spawner.getInitialSpawns() : spawner.getSpawns();
+        KList<IrisEntitySpawn> s = spawner.getInitialSpawns();
         if (s.isEmpty()) {
             return;
         }

core/src/main/java/com/volmit/iris/engine/object/IrisEngineService.java

@@ -0,0 +1,19 @@
+package com.volmit.iris.engine.object;
+
+import com.volmit.iris.Iris;
+import com.volmit.iris.engine.framework.Engine;
+import lombok.RequiredArgsConstructor;
+import org.bukkit.event.Listener;
+
+@RequiredArgsConstructor
+public abstract class IrisEngineService implements Listener {
+    protected final Engine engine;
+
+    public abstract void onEnable(boolean hotload);
+
+    public abstract void onDisable(boolean hotload);
+
+    public final void postShutdown(Runnable r) {
+        Iris.instance.postShutdown(r);
+    }
+}

core/src/main/java/com/volmit/iris/engine/object/IrisEntity.java

@@ -404,7 +404,7 @@ public class IrisEntity extends IrisRegistrant {
         });
 
 
-        return e;
+        return e.isValid() ? e : null;
     }
 
     private int surfaceY(Location l) {

core/src/main/java/com/volmit/iris/engine/object/IrisEntitySpawn.java

@@ -28,19 +28,12 @@ import com.volmit.iris.util.math.RNG;
 import com.volmit.iris.util.math.Vector3d;
 import com.volmit.iris.util.matter.MatterMarker;
 import com.volmit.iris.util.matter.slices.MarkerMatter;
-import io.lumine.mythic.bukkit.adapters.BukkitEntity;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.experimental.Accessors;
-import org.bukkit.Chunk;
+import org.bukkit.*;
-import org.bukkit.Location;
-import org.bukkit.Material;
-import org.bukkit.World;
 import org.bukkit.entity.Entity;
-import org.bukkit.entity.EntityType;
-import org.bukkit.entity.LivingEntity;
-import org.bukkit.util.BoundingBox;
 
 @Snippet("entity-spawn")
 @Accessors(chain = true)
@@ -69,6 +62,10 @@ public class IrisEntitySpawn implements IRare {
     private transient IrisSpawner referenceSpawner;
     private transient IrisMarker referenceMarker;
 
+    public boolean check(Engine eng, IrisPosition c, ChunkSnapshot snapshot) {
+        return getRealEntity(eng).getSurface().matches(snapshot.getBlockData(c.getX() & 15, c.getY(), c.getZ() & 15));
+    }
+
     public int spawn(Engine gen, Chunk c, RNG rng) {
         int spawns = minSpawns == maxSpawns ? minSpawns : rng.i(Math.min(minSpawns, maxSpawns), Math.max(minSpawns, maxSpawns));
         int s = 0;
@@ -168,7 +165,7 @@ public class IrisEntitySpawn implements IRare {
                 return null;
             }
 
-            if (!ignoreSurfaces && !irisEntity.getSurface().matches(at.clone().subtract(0, 1, 0).getBlock())) {
+            if (!ignoreSurfaces && !irisEntity.getSurface().matches(at.clone().subtract(0, 1, 0).getBlock().getBlockData())) {
                 return null;
             }
 
@@ -183,6 +180,10 @@ public class IrisEntitySpawn implements IRare {
             Entity e = irisEntity.spawn(g, at.add(0.5, 0.5, 0.5), rng.aquire(() -> new RNG(g.getSeedManager().getEntity())));
             if (e != null) {
                 Iris.debug("Spawned " + C.DARK_AQUA + "Entity<" + getEntity() + "> " + C.GREEN + e.getType() + C.LIGHT_PURPLE + " @ " + C.GRAY + e.getLocation().getX() + ", " + e.getLocation().getY() + ", " + e.getLocation().getZ());
+
+                if (referenceSpawner != null) {
+                    referenceSpawner.getConditions().apply(e);
+                }
             }
 
             return e;

core/src/main/java/com/volmit/iris/engine/object/IrisSpawnCondition.java

@@ -0,0 +1,58 @@
+package com.volmit.iris.engine.object;
+
+import com.volmit.iris.Iris;
+import com.volmit.iris.engine.object.annotations.Desc;
+import com.volmit.iris.util.collection.KList;
+import com.volmit.iris.util.collection.KMap;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.Accessors;
+import org.bukkit.NamespacedKey;
+import org.bukkit.entity.Entity;
+import org.bukkit.entity.SpawnCategory;
+import org.bukkit.persistence.PersistentDataType;
+
+import java.util.UUID;
+import java.util.function.BooleanSupplier;
+
+@Accessors(chain = true)
+@NoArgsConstructor
+@AllArgsConstructor
+@Desc("Conditions for a spawner to be triggered")
+@Data
+public class IrisSpawnCondition {
+    private static final NamespacedKey CATEGORY_KEY = new NamespacedKey(Iris.instance, "spawn_category");
+
+    private SpawnCategory category = SpawnCategory.AMBIENT;
+    private int maxEntities = 60;
+
+    public boolean check(KMap<UUID, KMap<String, Boolean>> cache, KList<Entity> entities) {
+        int entityCount = 0;
+        for (Entity entity : entities) {
+            var map = cache.computeIfAbsent(entity.getUniqueId(), k -> new KMap<>());
+            if (check(map, "category_" + category.name(), () -> checkCategory(entity, category)) && ++entityCount >= maxEntities)
+                return false;
+        }
+        return true;
+    }
+
+    public void apply(Entity entity) {
+        var pdc = entity.getPersistentDataContainer();
+        pdc.set(CATEGORY_KEY, PersistentDataType.STRING, category.name());
+    }
+
+    private static boolean check(KMap<String, Boolean> cache, String key, BooleanSupplier predicate) {
+        return cache.computeIfAbsent(key, k -> predicate.getAsBoolean()) == Boolean.TRUE;
+    }
+
+    private static boolean checkCategory(Entity entity, SpawnCategory category) {
+        if (entity.getSpawnCategory() == category)
+            return true;
+
+        var pdc = entity.getPersistentDataContainer();
+        if (!pdc.has(CATEGORY_KEY, PersistentDataType.STRING))
+            return false;
+        return category.name().equals(pdc.get(CATEGORY_KEY, PersistentDataType.STRING));
+    }
+}

core/src/main/java/com/volmit/iris/engine/object/IrisSpawner.java

@@ -74,6 +74,9 @@ public class IrisSpawner extends IrisRegistrant {
     @Desc("Where should these spawns be placed")
     private IrisSpawnGroup group = IrisSpawnGroup.NORMAL;
 
+    @Desc("Conditions for this spawner to be triggered")
+    private IrisSpawnCondition conditions = new IrisSpawnCondition();
+
     public boolean isValid(IrisBiome biome) {
         return switch (group) {
             case NORMAL -> switch (biome.getInferredType()) {

core/src/main/java/com/volmit/iris/engine/object/IrisSurface.java

@@ -20,7 +20,7 @@ package com.volmit.iris.engine.object;
 
 import com.volmit.iris.engine.object.annotations.Desc;
 import org.bukkit.Material;
-import org.bukkit.block.Block;
+import org.bukkit.block.data.BlockData;
 import org.bukkit.block.data.Waterlogged;
 
 @Desc("The type of surface entities should spawn on")
@@ -47,8 +47,8 @@ public enum IrisSurface {
      * @param state The blockstate
      * @return True if it matches
      */
-    public boolean matches(Block state) {
+    public boolean matches(BlockData state) {
-        Material type = state.getType();
+        Material type = state.getMaterial();
         if (type.isSolid()) {
             return this == LAND || this == OVERWORLD || (this == ANIMAL
                     && (type == Material.GRASS_BLOCK || type == Material.DIRT

core/src/main/java/com/volmit/iris/engine/service/EngineMobHandlerSVC.java

@@ -0,0 +1,340 @@
+package com.volmit.iris.engine.service;
+
+import com.google.common.util.concurrent.AtomicDouble;
+import com.volmit.iris.Iris;
+import com.volmit.iris.core.IrisSettings;
+import com.volmit.iris.core.loader.IrisData;
+import com.volmit.iris.core.nms.container.Pair;
+import com.volmit.iris.engine.IrisWorldManager;
+import com.volmit.iris.engine.framework.Engine;
+import com.volmit.iris.engine.object.*;
+import com.volmit.iris.util.collection.KList;
+import com.volmit.iris.util.collection.KMap;
+import com.volmit.iris.util.collection.KSet;
+import com.volmit.iris.util.format.Form;
+import com.volmit.iris.util.mantle.Mantle;
+import com.volmit.iris.util.math.BlockPosition;
+import com.volmit.iris.util.math.Position2;
+import com.volmit.iris.util.math.RNG;
+import com.volmit.iris.util.matter.MatterMarker;
+import com.volmit.iris.util.parallel.Sync;
+import com.volmit.iris.util.scheduling.J;
+import com.volmit.iris.util.scheduling.PrecisionStopwatch;
+import io.papermc.lib.PaperLib;
+import lombok.SneakyThrows;
+import org.bukkit.ChunkSnapshot;
+import org.bukkit.GameRule;
+import org.bukkit.World;
+import org.bukkit.entity.Entity;
+import org.bukkit.entity.Player;
+import org.bukkit.event.EventHandler;
+import org.bukkit.event.EventPriority;
+import org.bukkit.event.player.PlayerChangedWorldEvent;
+import org.bukkit.event.player.PlayerJoinEvent;
+import org.bukkit.event.player.PlayerQuitEvent;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class EngineMobHandlerSVC extends IrisEngineService {
+    private static final List<String> CAVE_TAGS = List.of("cave_floor", "cave_ceiling");
+    private static final int SAFE_RADIUS = 16;
+    private static final int MAX_RADIUS = 128;
+
+    private final AtomicLong currentTick = new AtomicLong();
+    private final Sync<Long> sync = new Sync<>();
+    private final Set<Player> players = ConcurrentHashMap.newKeySet();
+    private final AtomicBoolean running = new AtomicBoolean(false);
+    private transient KList<Entity> entities = new KList<>();
+    private transient Thread asyncTicker = null;
+    private transient Thread entityCollector = null;
+    private transient boolean charge = false;
+    private transient int task = -1;
+
+    public EngineMobHandlerSVC(Engine engine) {
+        super(engine);
+    }
+
+    @Override
+    public void onEnable(boolean hotload) {
+        if (running.get()) {
+            running.set(false);
+            cancel(asyncTicker);
+            cancel(entityCollector);
+        }
+
+        running.set(true);
+        charge = hotload;
+        asyncTicker = Thread.ofPlatform()
+                .name("Iris Async Mob Spawning - " + engine.getWorld().name())
+                .priority(9)
+                .start(() -> {
+                    while (mayLoop()) {
+                        try {
+                            asyncTick();
+                        } catch (Throwable e) {
+                            if (isInterrupted(e))
+                                return;
+                            Iris.error("Error in async tick for " + engine.getWorld().name());
+                            e.printStackTrace();
+
+                            J.sleep(100);
+                        }
+                    }
+                });
+        entityCollector = Thread.ofVirtual()
+                .name("Iris Async Entity Collector - " + engine.getWorld().name())
+                .start(() -> {
+                    while (mayLoop()) {
+                        try {
+                            sync.next().get();
+                            var world = engine.getWorld().realWorld();
+                            if (world == null) continue;
+                            J.s(() -> entities = new KList<>(world.getEntities()));
+                        } catch (Throwable e) {
+                            if (isInterrupted(e))
+                                return;
+
+                            Iris.error("Error in async tick for " + engine.getWorld().name());
+                            e.printStackTrace();
+
+                            J.sleep(100);
+                        }
+                    }
+                });
+        if (task != -1) J.csr(task);
+        task = J.sr(() -> sync.advance(currentTick.getAndIncrement()), 0);
+    }
+
+    @Override
+    public void onDisable(boolean hotload) {
+        running.set(false);
+        cancel(asyncTicker);
+        cancel(entityCollector);
+        if (!hotload) J.csr(task);
+    }
+
+    private void asyncTick() throws Throwable {
+        long tick = sync.next().get();
+        var manager = (IrisWorldManager) engine.getWorldManager();
+        if (charge) {
+            manager.chargeEnergy();
+            charge = false;
+        }
+
+        var world = engine.getWorld().realWorld();
+        if (world == null
+                || noSpawning()
+                || Boolean.FALSE.equals(world.getGameRuleValue(GameRule.DO_MOB_SPAWNING))
+                || players.isEmpty()
+                || manager.getEnergy() < 100)
+            return;
+
+        var p = PrecisionStopwatch.start();
+        var entities = new KList<>(this.entities);
+
+        var conditionCache = new KMap<UUID, KMap<String, Boolean>>();
+        var data = engine.getData();
+        var invalid = data.getSpawnerLoader()
+                .streamAllPossible()
+                .filter(Predicate.not(spawner -> spawner.canSpawn(engine)
+                        && spawner.getConditions().check(conditionCache, entities)))
+                .map(IrisSpawner::getLoadKey)
+                .collect(Collectors.toSet());
+
+        var centers = players.stream()
+                .filter(Objects::nonNull)
+                .filter(Player::isOnline)
+                .map(Player::getLocation)
+                .map(BlockPosition::fromLocation)
+                .collect(KList.collector())
+                .shuffle();
+
+        if (centers.isEmpty())
+            return;
+
+        AtomicDouble delta = new AtomicDouble();
+        int actuallySpawned = 0;
+
+        KMap<Position2, Pair<Entity[], ChunkSnapshot>> cache = new KMap<>();
+        while (centers.isNotEmpty()) {
+            var center = centers.pop();
+            var spawned = trySpawn(world, invalid, cache, center, delta);
+            if (spawned == 0 && p.getMilliseconds() < 1000)
+                centers.add(center);
+            actuallySpawned += spawned;
+        }
+        manager.setEnergy(manager.getEnergy() - delta.get());
+        if (actuallySpawned > 0) {
+            Iris.info("Async Mob Spawning " + world.getName() + " used " + delta + " energy and took " + Form.duration((long) p.getMilliseconds()));
+        }
+    }
+
+    private int trySpawn(
+            World world,
+            Set<String> invalid,
+            KMap<Position2, Pair<Entity[], ChunkSnapshot>> cache,
+            BlockPosition center,
+            AtomicDouble delta
+    ) {
+        var pos = center.randomPoint(MAX_RADIUS, SAFE_RADIUS);
+        if (pos.getY() < world.getMinHeight() || pos.getY() >= world.getMaxHeight())
+            return 0;
+
+        var chunkPos = new Position2(center.getX() >> 4, center.getZ() >> 4);
+        var pair = cache.computeIfAbsent(chunkPos, cPos -> {
+            try {
+                return PaperLib.getChunkAtAsync(world, cPos.getX(), cPos.getZ(), false)
+                        .thenApply(c -> c != null ? new Pair<>(c.getEntities(), c.getChunkSnapshot(false, false, false)) : null)
+                        .get();
+            } catch (InterruptedException | ExecutionException e) {
+                throw new RuntimeException(e);
+            }
+        });
+        if (pair == null)
+            return 0;
+
+        var spawners = spawnersAt(pair.getB(), pos, invalid);
+        spawners.removeIf(i -> !i.canSpawn(engine, chunkPos.getX(), chunkPos.getZ()));
+
+        if (spawners.isEmpty())
+            return 0;
+
+        IrisPosition irisPos = new IrisPosition(pos.getX(), pos.getY(), pos.getZ());
+        for (var spawner : spawners) {
+            var spawns = spawner.getSpawns().copy();
+            spawns.removeIf(spawn -> !spawn.check(engine, irisPos, pair.getB()));
+
+            var entity = IRare.pick(spawns, RNG.r.nextDouble());
+            if (entity == null)
+                continue;
+
+            entity.setReferenceSpawner(spawner);
+            entity.setReferenceMarker(spawner.getReferenceMarker());
+            int spawned = entity.spawn(engine, irisPos, RNG.r);
+            if (spawned == 0)
+                continue;
+
+            delta.addAndGet(spawned * ((entity.getEnergyMultiplier() * spawner.getEnergyMultiplier() * 1)));
+            spawner.spawn(engine, chunkPos.getX(), chunkPos.getZ());
+            if (!spawner.canSpawn(engine))
+                invalid.add(spawner.getLoadKey());
+            return spawned;
+        }
+        return 0;
+    }
+
+    private KSet<IrisSpawner> spawnersAt(ChunkSnapshot chunk, BlockPosition pos, Set<String> invalid) {
+        KSet<IrisSpawner> spawners = markerAt(chunk, pos, invalid);
+
+        var loader = engine.getData().getSpawnerLoader();
+        int y = pos.getY() - engine.getWorld().minHeight();
+        Stream.concat(engine.getRegion(pos.getX(), pos.getZ())
+                                .getEntitySpawners()
+                                .stream(),
+                        engine.getBiomeOrMantle(pos.getX(), y, pos.getZ())
+                                .getEntitySpawners()
+                                .stream())
+                .filter(Predicate.not(invalid::contains))
+                .map(loader::load)
+                .forEach(spawners::add);
+
+        return spawners;
+    }
+
+    private KSet<IrisSpawner> markerAt(ChunkSnapshot chunk, BlockPosition pos, Set<String> invalid) {
+        if (!IrisSettings.get().getWorld().isMarkerEntitySpawningSystem())
+            return new KSet<>();
+
+        int y = pos.getY() - engine.getWorld().minHeight();
+        Mantle mantle = engine.getMantle().getMantle();
+        MatterMarker matter = mantle.get(pos.getX(), y, pos.getZ(), MatterMarker.class);
+        if (matter == null || CAVE_TAGS.contains(matter.getTag()))
+            return new KSet<>();
+        IrisData data = engine.getData();
+        IrisMarker mark = data.getMarkerLoader().load(matter.getTag());
+        if (mark == null)
+            return new KSet<>();
+
+        if (mark.isEmptyAbove()) {
+            int x = pos.getX() & 15, z = pos.getZ() & 15;
+            boolean remove = chunk.getBlockData(x, pos.getY() + 1, z).getMaterial().isSolid() || chunk.getBlockData(x, pos.getY() + 2, z).getMaterial().isSolid();
+            if (remove) {
+                mantle.remove(pos.getX(), y, pos.getZ(), MatterMarker.class);
+                return new KSet<>();
+            }
+        }
+
+        KSet<IrisSpawner> spawners = new KSet<>();
+        for (String key : mark.getSpawners()) {
+            if (invalid.contains(key))
+                continue;
+
+            IrisSpawner spawner = data.getSpawnerLoader().load(key);
+            if (spawner == null) {
+                Iris.error("Cannot load spawner: " + key + " for marker " + matter.getTag());
+                continue;
+            }
+
+            spawner.setReferenceMarker(mark);
+            spawners.add(spawner);
+        }
+        return spawners;
+    }
+
+    @EventHandler(priority = EventPriority.LOWEST)
+    public void on(PlayerJoinEvent event) {
+        var player = event.getPlayer();
+        if (player.getWorld() != engine.getWorld().realWorld())
+            return;
+        players.add(player);
+    }
+
+    @EventHandler(priority = EventPriority.LOWEST)
+    public void on(PlayerQuitEvent event) {
+        players.remove(event.getPlayer());
+    }
+
+    @EventHandler(priority = EventPriority.MONITOR, ignoreCancelled = true)
+    public void on(PlayerChangedWorldEvent event) {
+        var player = event.getPlayer();
+        if (player.getWorld() == engine.getWorld().realWorld())
+            players.add(player);
+        else
+            players.remove(player);
+    }
+
+    @SneakyThrows
+    private static void cancel(Thread thread) {
+        if (thread == null || !thread.isAlive()) return;
+        thread.interrupt();
+        thread.join();
+    }
+
+    private static boolean noSpawning() {
+        var world = IrisSettings.get().getWorld();
+        return !world.isMarkerEntitySpawningSystem() && !world.isAnbientEntitySpawningSystem();
+    }
+
+    private boolean mayLoop() {
+        return !engine.isClosed() && running.get() && !Thread.interrupted();
+    }
+
+    private static boolean isInterrupted(Throwable e) {
+        while (e != null) {
+            if (e instanceof InterruptedException)
+                return true;
+            e = e.getCause();
+        }
+        return false;
+    }
+}

core/src/main/java/com/volmit/iris/engine/service/EngineStatusSVC.java

@@ -0,0 +1,65 @@
+package com.volmit.iris.engine.service;
+
+import com.volmit.iris.engine.framework.Engine;
+import com.volmit.iris.engine.object.IrisEngineService;
+import com.volmit.iris.util.collection.KList;
+
+public class EngineStatusSVC extends IrisEngineService {
+    private static final KList<EngineStatusSVC> INSTANCES = new KList<>();
+
+    public EngineStatusSVC(Engine engine) {
+        super(engine);
+    }
+
+    @Override
+    public void onEnable(boolean hotload) {
+        if (hotload) return;
+        synchronized (INSTANCES) {
+            INSTANCES.add(this);
+        }
+    }
+
+    @Override
+    public void onDisable(boolean hotload) {
+        if (hotload) return;
+
+        synchronized (INSTANCES) {
+            INSTANCES.remove(this);
+        }
+    }
+
+    public static int getEngineCount() {
+        return INSTANCES.size();
+    }
+
+    public static Status getStatus() {
+        synchronized (INSTANCES) {
+            long loadedChunks = 0;
+            long tectonicPlates = 0;
+            long activeTectonicPlates = 0;
+            long queuedTectonicPlates = 0;
+            long minTectonicUnloadDuration = Long.MAX_VALUE;
+            long maxTectonicUnloadDuration = Long.MIN_VALUE;
+
+            for (var service : INSTANCES) {
+                var world = service.engine.getWorld();
+                if (world.hasRealWorld())
+                    loadedChunks += world.realWorld().getLoadedChunks().length;
+
+                tectonicPlates += service.engine.getMantle().getLoadedRegionCount();
+                activeTectonicPlates += service.engine.getMantle().getNotQueuedLoadedRegions();
+                queuedTectonicPlates += service.engine.getMantle().getToUnload();
+                minTectonicUnloadDuration = Math.min(minTectonicUnloadDuration, (long) service.engine.getMantle().getTectonicDuration());
+                maxTectonicUnloadDuration = Math.max(maxTectonicUnloadDuration, (long) service.engine.getMantle().getTectonicDuration());
+            }
+            return new Status(INSTANCES.size(), loadedChunks, MantleCleanerSVC.getTectonicLimit(), tectonicPlates, activeTectonicPlates, queuedTectonicPlates, minTectonicUnloadDuration, maxTectonicUnloadDuration);
+        }
+    }
+
+    public record Status(int engineCount, long loadedChunks, int tectonicLimit,
+                         long tectonicPlates, long activeTectonicPlates,
+                         long queuedTectonicPlates,
+                         long minTectonicUnloadDuration,
+                         long maxTectonicUnloadDuration) {
+    }
+}

core/src/main/java/com/volmit/iris/engine/service/MantleCleanerSVC.java

@@ -0,0 +1,127 @@
+package com.volmit.iris.engine.service;
+
+import com.volmit.iris.Iris;
+import com.volmit.iris.core.IrisSettings;
+import com.volmit.iris.engine.framework.Engine;
+import com.volmit.iris.engine.object.IrisEngineService;
+import com.volmit.iris.util.format.C;
+import com.volmit.iris.util.math.M;
+import com.volmit.iris.util.misc.getHardware;
+import com.volmit.iris.util.scheduling.Looper;
+import lombok.SneakyThrows;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.LongSupplier;
+
+public class MantleCleanerSVC extends IrisEngineService {
+    private static final AtomicInteger tectonicLimit = new AtomicInteger(30);
+
+    static {
+        // todo: Redo this
+        tectonicLimit.set(2);
+        long t = getHardware.getProcessMemory();
+        while (t > 200) {
+            tectonicLimit.incrementAndGet();
+            t = t - 200;
+        }
+    }
+
+    private Ticker trimmer;
+    private Ticker unloader;
+
+    public MantleCleanerSVC(Engine engine) {
+        super(engine);
+    }
+
+    public static int getTectonicLimit() {
+        return tectonicLimit.get();
+    }
+
+    private static Ticker createTrimmer(Engine engine) {
+        return new Ticker(() -> {
+            if (engine.isClosed()) return -1;
+            long start = M.ms();
+            try {
+                engine.getMantle().trim(tectonicLimit.get() / getEngineCount());
+            } catch (Throwable e) {
+                Iris.debug(C.RED + "Mantle: Failed to trim.");
+                Iris.reportError(e);
+                e.printStackTrace();
+            }
+
+            if (engine.isClosed()) return -1;
+            int size = getEngineCount();
+            return Math.max(1000 / size - (M.ms() - start), 0);
+        }, "Iris Mantle Trimmer - " + engine.getWorld().name());
+    }
+
+    private static Ticker createUnloader(Engine engine) {
+        return new Ticker(() -> {
+            if (engine.isClosed()) return -1;
+            long start = M.ms();
+            try {
+                engine.getMantle().unloadTectonicPlate(tectonicLimit.get() / getEngineCount());
+            } catch (Throwable e) {
+                Iris.debug(C.RED + "Mantle: Failed to unload.");
+                Iris.reportError(e);
+                e.printStackTrace();
+            }
+
+            if (engine.isClosed()) return -1;
+            int size = getEngineCount();
+            return Math.max(1000 / size - (M.ms() - start), 0);
+        }, "Iris Mantle Unloader - " + engine.getWorld().name());
+    }
+
+    private static int getEngineCount() {
+        return Math.max(EngineStatusSVC.getEngineCount(), 1);
+    }
+
+    @Override
+    public void onEnable(boolean hotload) {
+        if (engine.isStudio() && !IrisSettings.get().getPerformance().trimMantleInStudio)
+            return;
+        if (trimmer == null || !trimmer.isAlive())
+            trimmer = createTrimmer(engine);
+        if (unloader == null || !unloader.isAlive())
+            unloader = createUnloader(engine);
+    }
+
+    @Override
+    public void onDisable(boolean hotload) {
+        if (hotload) return;
+        if (trimmer != null) trimmer.await();
+        if (unloader != null) unloader.await();
+    }
+
+    private static class Ticker extends Looper {
+        private final LongSupplier supplier;
+
+        private Ticker(LongSupplier supplier, String name) {
+            this.supplier = supplier;
+            setPriority(Thread.MIN_PRIORITY);
+            setName(name);
+            start();
+        }
+
+        @Override
+        protected long loop() {
+            try {
+                return supplier.getAsLong();
+            } catch (Throwable e) {
+                Iris.error("Exception in Looper " + getName());
+                StringWriter sw = new StringWriter();
+                e.printStackTrace(new PrintWriter(sw));
+                Iris.error(sw.toString());
+                return 3000;
+            }
+        }
+
+        @SneakyThrows
+        public void await() {
+            join();
+        }
+    }
+}

core/src/main/java/com/volmit/iris/util/mantle/Mantle.java

@@ -21,7 +21,6 @@ package com.volmit.iris.util.mantle;
 import com.google.common.util.concurrent.AtomicDouble;
 import com.volmit.iris.Iris;
 import com.volmit.iris.core.IrisSettings;
-import com.volmit.iris.core.service.IrisEngineSVC;
 import com.volmit.iris.core.tools.IrisToolbelt;
 import com.volmit.iris.engine.data.cache.Cache;
 import com.volmit.iris.engine.framework.Engine;
@@ -425,7 +424,7 @@ public class Mantle {
         ioTrim.set(true);
         unloadLock.lock();
         try {
-            if (lastUse != null && IrisEngineSVC.instance != null) {
+            if (lastUse != null) {
                 if (!lastUse.isEmpty()) {
                     Iris.debug("Trimming Tectonic Plates older than " + Form.duration(adjustedIdleDuration.get(), 0));
                     for (long i : new ArrayList<>(lastUse.keySet())) {
@@ -435,7 +434,6 @@ public class Mantle {
                             if (lastUseTime != null && M.ms() - lastUseTime >= finalAdjustedIdleDuration) {
                                 toUnload.add(i);
                                 Iris.debug("Tectonic Region added to unload");
-                                IrisEngineSVC.instance.trimActiveAlive.reset();
                             }
                         });
                     }
@@ -454,53 +452,49 @@ public class Mantle {
         AtomicInteger i = new AtomicInteger();
         unloadLock.lock();
         BurstExecutor burst = null;
-        if (IrisEngineSVC.instance != null) {
+        try {
-            try {
+            KList<Long> copy = toUnload.copy();
-                KList<Long> copy = toUnload.copy();
+            if (!disableClear) toUnload.clear();
-                if (!disableClear) toUnload.clear();
+            burst = MultiBurst.burst.burst(copy.size());
-                burst = MultiBurst.burst.burst(copy.size());
+            burst.setMulticore(copy.size() > tectonicLimit);
-                burst.setMulticore(copy.size() > tectonicLimit);
+            for (int j = 0; j < copy.size(); j++) {
-                for (int j = 0; j < copy.size(); j++) {
+                Long id = copy.get(j);
-                    Long id = copy.get(j);
+                if (id == null) {
-                    if (id == null) {
+                    Iris.error("Null id in unloadTectonicPlate at index " + j);
-                        Iris.error("Null id in unloadTectonicPlate at index " + j);
+                    continue;
-                        continue;
-                    }
-
-                    burst.queue(() ->
-                            hyperLock.withLong(id, () -> {
-                                TectonicPlate m = loadedRegions.get(id);
-                                if (m != null) {
-                                    if (m.inUse()) {
-                                        Iris.debug("Tectonic Plate was added to unload while in use " + C.DARK_GREEN + m.getX() + " " + m.getZ());
-                                        if (disableClear) toUnload.remove(id);
-                                        lastUse.put(id, M.ms());
-                                        return;
-                                    }
-                                    try {
-                                        m.write(fileForRegion(dataFolder, id));
-                                        loadedRegions.remove(id);
-                                        lastUse.remove(id);
-                                        if (disableClear) toUnload.remove(id);
-                                        i.incrementAndGet();
-                                        Iris.debug("Unloaded Tectonic Plate " + C.DARK_GREEN + Cache.keyX(id) + " " + Cache.keyZ(id));
-                                        IrisEngineSVC.instance.unloadActiveAlive.reset();
-                                    } catch (IOException e) {
-                                        Iris.reportError(e);
-                                    }
-                                }
-                            }));
                 }
-                burst.complete();
+
-            } catch (Throwable e) {
+                burst.queue(() ->
-                e.printStackTrace();
+                        hyperLock.withLong(id, () -> {
-                if (burst != null)
+                            TectonicPlate m = loadedRegions.get(id);
-                    burst.complete();
+                            if (m != null) {
-            } finally {
+                                if (m.inUse()) {
-                unloadLock.unlock();
+                                    Iris.debug("Tectonic Plate was added to unload while in use " + C.DARK_GREEN + m.getX() + " " + m.getZ());
-                ioTectonicUnload.set(true);
+                                    if (disableClear) toUnload.remove(id);
+                                    lastUse.put(id, M.ms());
+                                    return;
+                                }
+                                try {
+                                    m.write(fileForRegion(dataFolder, id));
+                                    loadedRegions.remove(id);
+                                    lastUse.remove(id);
+                                    if (disableClear) toUnload.remove(id);
+                                    i.incrementAndGet();
+                                    Iris.debug("Unloaded Tectonic Plate " + C.DARK_GREEN + Cache.keyX(id) + " " + Cache.keyZ(id));
+                                } catch (IOException e) {
+                                    Iris.reportError(e);
+                                }
+                            }
+                        }));
             }
-            return i.get();
+            burst.complete();
+        } catch (Throwable e) {
+            e.printStackTrace();
+            if (burst != null)
+                burst.complete();
+        } finally {
+            unloadLock.unlock();
+            ioTectonicUnload.set(true);
         }
         return i.get();
     }

core/src/main/java/com/volmit/iris/util/math/BlockPosition.java

@@ -19,11 +19,14 @@
 package com.volmit.iris.util.math;
 
 import lombok.Data;
+import org.bukkit.Location;
 import org.bukkit.World;
 import org.bukkit.block.Block;
 
 import java.util.Objects;
 
+import static com.volmit.iris.util.math.RNG.r;
+
 @Data
 public class BlockPosition {
     //Magic numbers
@@ -43,6 +46,10 @@ public class BlockPosition {
         this.z = z;
     }
 
+    public static BlockPosition fromLocation(Location loc) {
+        return new BlockPosition(loc.getBlockX(), loc.getBlockY(), loc.getBlockZ());
+    }
+
     public static long toLong(int x, int y, int z) {
         long var3 = 0L;
         var3 |= (x & m4) << m3;
@@ -108,4 +115,18 @@ public class BlockPosition {
         setY(Math.max(i.getY(), getY()));
         setZ(Math.max(i.getZ(), getZ()));
     }
+
+    public BlockPosition randomPoint(int radius, int innerRadius) {
+        int max = radius * radius;
+        int min = innerRadius * innerRadius;
+
+        while (true) {
+            int x = r.nextInt(-radius, radius + 1);
+            int y = r.nextInt(-radius, radius + 1);
+            int z = r.nextInt(-radius, radius + 1);
+            double dist = x * x + y * y + z * z;
+            if (dist < min || dist > max) continue;
+            return add(x, y, z);
+        }
+    }
 }

core/src/main/java/com/volmit/iris/util/parallel/Sync.java

@@ -0,0 +1,20 @@
+package com.volmit.iris.util.parallel;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class Sync<T> {
+    private final AtomicReference<CompletableFuture<T>> tick = new AtomicReference<>(new CompletableFuture<>());
+
+    public void cancel(boolean mayInterruptIfRunning) {
+        tick.getAndSet(new CompletableFuture<>()).cancel(mayInterruptIfRunning);
+    }
+
+    public CompletableFuture<T> next() {
+        return tick.get();
+    }
+
+    public void advance(T value) {
+        tick.getAndSet(new CompletableFuture<>()).complete(value);
+    }
+}

nms/v1_21_R3/src/main/java/com/volmit/iris/core/nms/v1_21_R3/IrisChunkGenerator.java

@@ -209,7 +209,7 @@ public class IrisChunkGenerator extends CustomChunkGenerator {
 
     @Override
     public WeightedRandomList<MobSpawnSettings.SpawnerData> getMobsAt(Holder<Biome> holder, StructureManager structuremanager, MobCategory enumcreaturetype, BlockPos blockposition) {
-        return delegate.getMobsAt(holder, structuremanager, enumcreaturetype, blockposition);
+        return WeightedRandomList.create();
     }
 
     @Override