Experimental Parallel World Ticking

PARALLEL_INCOMPATIBLE_PLUGINS.md

@@ -0,0 +1,135 @@
+# Plugins incompatible with Parallel World Ticking
+
+A list of plugins that I found out that causes issues with Parallel World Ticking.
+
+This is not a full comprehensive list because I only use a few public plugins, most of the plugins I use on SparklyPower are made by myself.
+
+## NoCheatPlus (Updated)
+
+The movement checks can crash the server due to concurrency issues (more below), disable them in the config.
+
+```javastacktrace
+[03:14:05] [serverlevel-tick-worker-8/ERROR]: THE SERVER IS GOING TO CRASH! - Thread serverlevel-tick-worker-8 failed main thread check: Cannot query another world's (world) chunk (25, 16) in a ServerLevelTickThread - Is tick thread? true; Is server level tick thread? true; Currently ticking level: ArenasPvP; Is iterating over levels? true; Are we going to hard throw? false
+java.lang.Throwable: null
+	at net.minecraft.server.level.ServerChunkCache.getChunk(ServerChunkCache.java:272) ~[?:?]
+	at net.minecraft.world.level.Level.getChunk(Level.java:900) ~[?:?]
+	at net.minecraft.world.level.Level.getBlockState(Level.java:1175) ~[?:?]
+	at org.bukkit.craftbukkit.v1_20_R2.block.CraftBlock.getType(CraftBlock.java:238) ~[sparklypaper-1.20.2.jar:git-SparklyPaper-"049a8e5"]
+	at fr.neatmonster.nocheatplus.compat.bukkit.BlockCacheBukkit.fetchTypeId(BlockCacheBukkit.java:52) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.utilities.map.BlockCache.getOrCreateNode(BlockCache.java:317) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.utilities.map.BlockCache.getType(BlockCache.java:379) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.utilities.map.BlockProperties.collectFlagsSimple(BlockProperties.java:4454) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.utilities.location.RichBoundsLocation.collectBlockFlags(RichBoundsLocation.java:1327) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.utilities.location.RichBoundsLocation.collectBlockFlags(RichBoundsLocation.java:1309) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.checks.moving.model.LocationData.setExtraProperties(LocationData.java:92) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.checks.moving.model.MoveData.setWithExtraProperties(MoveData.java:207) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.checks.moving.MovingData.resetPlayerPositions(MovingData.java:585) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.checks.moving.util.AuxMoving.resetPositionsAndMediumProperties(AuxMoving.java:116) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.checks.moving.MovingListener.onPlayerTeleportMonitor(MovingListener.java:1989) ~[NoCheatPlus.jar:?]
+	at jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103) ~[?:?]
+	at java.lang.reflect.Method.invoke(Method.java:580) ~[?:?]
+	at fr.neatmonster.nocheatplus.event.mini.MultiListenerRegistry$AutoListener.onEvent(MultiListenerRegistry.java:82) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.event.mini.MiniListenerNode.onEvent(MiniListenerNode.java:157) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.event.mini.EventRegistryBukkit$4.execute(EventRegistryBukkit.java:124) ~[NoCheatPlus.jar:?]
+	at co.aikar.timings.TimedEventExecutor.execute(TimedEventExecutor.java:77) ~[sparklypaper-api-1.20.2-R0.1-SNAPSHOT.jar:git-SparklyPaper-"049a8e5"]
+	at org.bukkit.plugin.RegisteredListener.callEvent(RegisteredListener.java:70) ~[sparklypaper-api-1.20.2-R0.1-SNAPSHOT.jar:?]
+	at io.papermc.paper.plugin.manager.PaperEventManager.callEvent(PaperEventManager.java:54) ~[sparklypaper-1.20.2.jar:git-SparklyPaper-"049a8e5"]
+	at io.papermc.paper.plugin.manager.PaperPluginManagerImpl.callEvent(PaperPluginManagerImpl.java:126) ~[sparklypaper-1.20.2.jar:git-SparklyPaper-"049a8e5"]
+	at org.bukkit.plugin.SimplePluginManager.callEvent(SimplePluginManager.java:615) ~[sparklypaper-api-1.20.2-R0.1-SNAPSHOT.jar:?]
+	at org.bukkit.craftbukkit.v1_20_R2.entity.CraftPlayer.teleport(CraftPlayer.java:1354) ~[sparklypaper-1.20.2.jar:git-SparklyPaper-"049a8e5"]
+	at org.bukkit.craftbukkit.v1_20_R2.entity.CraftPlayer.teleport(CraftPlayer.java:1252) ~[sparklypaper-1.20.2.jar:git-SparklyPaper-"049a8e5"]
+	at net.perfectdreams.dreamcore.utils.extensions.EntityExtensionsKt.teleportToServerSpawn(EntityExtensions.kt:13) ~[DreamCore-reobf.jar:?]
+	at net.perfectdreams.dreamcore.utils.extensions.EntityExtensionsKt.teleportToServerSpawnWithEffects(EntityExtensions.kt:20) ~[DreamCore-reobf.jar:?]
+	at net.perfectdreams.dreamcore.utils.extensions.EntityExtensionsKt.teleportToServerSpawnWithEffects$default(EntityExtensions.kt:19) ~[DreamCore-reobf.jar:?]
+	at net.perfectdreams.dreamxizum.utils.ArenaXizum.finishArena(ArenaXizum.kt:185) ~[DreamXizum-reobf.jar:?]
+	at net.perfectdreams.dreamxizum.listeners.XizumListener.onDeath(XizumListener.kt:55) ~[DreamXizum-reobf.jar:?]
+	at com.destroystokyo.paper.event.executor.asm.generated.GeneratedEventExecutor732.execute(Unknown Source) ~[?:?]
+	at org.bukkit.plugin.EventExecutor$2.execute(EventExecutor.java:77) ~[sparklypaper-api-1.20.2-R0.1-SNAPSHOT.jar:?]
+	at co.aikar.timings.TimedEventExecutor.execute(TimedEventExecutor.java:77) ~[sparklypaper-api-1.20.2-R0.1-SNAPSHOT.jar:git-SparklyPaper-"049a8e5"]
+	at org.bukkit.plugin.RegisteredListener.callEvent(RegisteredListener.java:70) ~[sparklypaper-api-1.20.2-R0.1-SNAPSHOT.jar:?]
+	at io.papermc.paper.plugin.manager.PaperEventManager.callEvent(PaperEventManager.java:54) ~[sparklypaper-1.20.2.jar:git-SparklyPaper-"049a8e5"]
+	at io.papermc.paper.plugin.manager.PaperPluginManagerImpl.callEvent(PaperPluginManagerImpl.java:126) ~[sparklypaper-1.20.2.jar:git-SparklyPaper-"049a8e5"]
+	at org.bukkit.plugin.SimplePluginManager.callEvent(SimplePluginManager.java:615) ~[sparklypaper-api-1.20.2-R0.1-SNAPSHOT.jar:?]
+	at org.bukkit.craftbukkit.v1_20_R2.event.CraftEventFactory.callPlayerDeathEvent(CraftEventFactory.java:984) ~[sparklypaper-1.20.2.jar:git-SparklyPaper-"049a8e5"]
+	at net.minecraft.server.level.ServerPlayer.die(ServerPlayer.java:961) ~[?:?]
+	at net.minecraft.world.entity.LivingEntity.hurt(LivingEntity.java:1548) ~[?:?]
+	at net.minecraft.world.entity.player.Player.hurt(Player.java:973) ~[?:?]
+	at net.minecraft.server.level.ServerPlayer.hurt(ServerPlayer.java:1130) ~[?:?]
+	at net.minecraft.world.entity.projectile.AbstractArrow.onHitEntity(AbstractArrow.java:402) ~[?:?]
+	at net.minecraft.world.entity.projectile.Projectile.onHit(Projectile.java:206) ~[?:?]
+	at net.minecraft.world.entity.projectile.Projectile.preOnHit(Projectile.java:197) ~[?:?]
+	at net.minecraft.world.entity.projectile.AbstractArrow.preOnHit(AbstractArrow.java:296) ~[?:?]
+	at net.minecraft.world.entity.projectile.AbstractArrow.tick(AbstractArrow.java:232) ~[?:?]
+	at net.minecraft.world.entity.projectile.Arrow.tick(Arrow.java:112) ~[?:?]
+	at net.minecraft.server.level.ServerLevel.tickNonPassenger(ServerLevel.java:1392) ~[?:?]
+	at net.minecraft.world.level.Level.guardEntityTick(Level.java:1314) ~[?:?]
+	at net.minecraft.server.level.ServerLevel.lambda$tick$8(ServerLevel.java:906) ~[?:?]
+	at net.minecraft.world.level.entity.EntityTickList.forEach(EntityTickList.java:49) ~[sparklypaper-1.20.2.jar:git-SparklyPaper-"049a8e5"]
+	at net.minecraft.server.level.ServerLevel.tick(ServerLevel.java:886) ~[?:?]
+	at net.minecraft.server.MinecraftServer.lambda$tickChildren$16(MinecraftServer.java:1600) ~[sparklypaper-1.20.2.jar:git-SparklyPaper-"049a8e5"]
+	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) ~[?:?]
+	at java.util.concurrent.FutureTask.run(FutureTask.java:317) ~[?:?]
+	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
+	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
+	at java.lang.Thread.run(Thread.java:1583) ~[?:?]
+```
+
+The Change Tracker also has a concurrency bug. NoCheatPlus uses `setAccess` to control which world is being used in `BlockCacheBukkit`. This is fine in a sequential ticking server, but in a parallel ticking server, there is a race condition where one server level tick thread changes the world to `World1`, then another thread changes the world to `World2`, then whoops, a crash happens!
+
+This also breaks the movement checks above: While the movement checks do have `synchronized` in their methods, the falling block checks don't, and both use the same BlockCache instance (handled by `setAccess`).
+
+https://github.com/Updated-NoCheatPlus/NoCheatPlus/blob/64bf374fd39297bab5b7adb646063debbd12643e/NCPCompatBukkit/src/main/java/fr/neatmonster/nocheatplus/compat/bukkit/BlockCacheBukkit.java#L52
+
+To work around this, disable `changetracker.active` and `changetracker.pistons`.
+
+I think that NoCheatPlus may have other concurrency issues too due to its use of `setAccess` around the code. In fact, I think that NoCheatPlus has the same issues in Folia too, even tho NoCheatPlus is marked as "Folia supported".
+
+```javastacktrace
+[03:35:04] [serverlevel-tick-worker-6/ERROR]: THE SERVER IS GOING TO CRASH! - Thread serverlevel-tick-worker-6 failed main thread check: Cannot query another world's (world) chunk (-817, -45) in a ServerLevelTickThread - Is tick thread? true; Is server level tick thread? true; Currently ticking level: Resources; Is iterating over levels? true; Are we going to hard throw? false
+java.lang.Throwable: null
+	at net.minecraft.server.level.ServerChunkCache.getChunk(ServerChunkCache.java:272) ~[?:?]
+	at net.minecraft.world.level.Level.getChunk(Level.java:900) ~[?:?]
+	at net.minecraft.world.level.Level.getBlockState(Level.java:1175) ~[?:?]
+	at org.bukkit.craftbukkit.v1_20_R2.block.CraftBlock.getType(CraftBlock.java:238) ~[sparklypaper-1.20.2.jar:git-SparklyPaper-"049a8e5"]
+	at fr.neatmonster.nocheatplus.compat.bukkit.BlockCacheBukkit.fetchTypeId(BlockCacheBukkit.java:52) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.utilities.map.BlockCache.getOrCreateNode(BlockCache.java:317) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.utilities.map.BlockCache.getOrCreateBlockCacheNode(BlockCache.java:442) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.compat.blocks.changetracker.BlockChangeTracker.addBlock(BlockChangeTracker.java:571) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.compat.blocks.changetracker.BlockChangeTracker.addBlocks(BlockChangeTracker.java:433) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.compat.blocks.changetracker.BlockChangeTracker.addBlocks(BlockChangeTracker.java:394) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.compat.blocks.changetracker.BlockChangeListener.onEntityChangeBlock(BlockChangeListener.java:300) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.compat.blocks.changetracker.BlockChangeListener.access$200(BlockChangeListener.java:56) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.compat.blocks.changetracker.BlockChangeListener$2.onEvent(BlockChangeListener.java:99) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.compat.blocks.changetracker.BlockChangeListener$2.onEvent(BlockChangeListener.java:93) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.event.mini.MiniListenerNode.onEvent(MiniListenerNode.java:157) ~[NoCheatPlus.jar:?]
+	at fr.neatmonster.nocheatplus.event.mini.EventRegistryBukkit$4.execute(EventRegistryBukkit.java:124) ~[NoCheatPlus.jar:?]
+	at co.aikar.timings.TimedEventExecutor.execute(TimedEventExecutor.java:77) ~[sparklypaper-api-1.20.2-R0.1-SNAPSHOT.jar:git-SparklyPaper-"049a8e5"]
+	at org.bukkit.plugin.RegisteredListener.callEvent(RegisteredListener.java:70) ~[sparklypaper-api-1.20.2-R0.1-SNAPSHOT.jar:?]
+	at io.papermc.paper.plugin.manager.PaperEventManager.callEvent(PaperEventManager.java:54) ~[sparklypaper-1.20.2.jar:git-SparklyPaper-"049a8e5"]
+	at io.papermc.paper.plugin.manager.PaperPluginManagerImpl.callEvent(PaperPluginManagerImpl.java:126) ~[sparklypaper-1.20.2.jar:git-SparklyPaper-"049a8e5"]
+	at org.bukkit.plugin.SimplePluginManager.callEvent(SimplePluginManager.java:615) ~[sparklypaper-api-1.20.2-R0.1-SNAPSHOT.jar:?]
+	at org.bukkit.craftbukkit.v1_20_R2.event.CraftEventFactory.callEntityChangeBlockEvent(CraftEventFactory.java:1411) ~[sparklypaper-1.20.2.jar:git-SparklyPaper-"049a8e5"]
+	at org.bukkit.craftbukkit.v1_20_R2.event.CraftEventFactory.callEntityChangeBlockEvent(CraftEventFactory.java:1403) ~[sparklypaper-1.20.2.jar:git-SparklyPaper-"049a8e5"]
+	at net.minecraft.world.entity.item.FallingBlockEntity.fall(FallingBlockEntity.java:98) ~[?:?]
+	at net.minecraft.world.entity.item.FallingBlockEntity.fall(FallingBlockEntity.java:92) ~[?:?]
+	at net.minecraft.world.level.block.FallingBlock.tick(FallingBlock.java:37) ~[?:?]
+	at net.minecraft.world.level.block.state.BlockBehaviour$BlockStateBase.tick(BlockBehaviour.java:1205) ~[?:?]
+	at net.minecraft.server.level.ServerLevel.tickBlock(ServerLevel.java:1340) ~[?:?]
+	at net.minecraft.world.ticks.LevelTicks.runCollectedTicks(LevelTicks.java:197) ~[?:?]
+	at net.minecraft.world.ticks.LevelTicks.tick(LevelTicks.java:94) ~[?:?]
+	at net.minecraft.server.level.ServerLevel.tick(ServerLevel.java:848) ~[?:?]
+	at net.minecraft.server.MinecraftServer.lambda$tickChildren$16(MinecraftServer.java:1600) ~[sparklypaper-1.20.2.jar:git-SparklyPaper-"049a8e5"]
+	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) ~[?:?]
+	at java.util.concurrent.FutureTask.run(FutureTask.java:317) ~[?:?]
+	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]
+	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]
+	at java.lang.Thread.run(Thread.java:1583) ~[?:?]
+```
+
+## MyPet
+
+If a player mounted on a Warden teleports to another world, the server crashes.
+
+This is caused by https://github.com/MyPetORG/MyPet/issues/1647 and can even cause issues in vanilla Paper. In vanilla Paper, instead of crashing the server, the player is teleported back to the Warden's location.
+
+Fork that removes the affecting code: https://github.com/SparklyPower/MyPet

PARALLEL_NOTES.md

@@ -0,0 +1,45 @@
+# Parallel World Ticking Notes
+
+Notes about the Parallel World Ticking implementation.
+
+i'm stoopid so this may have a lot of dumb incorrect stuff pls don't hurt me spottedleaf :(
+
+## Opening an inventory after a world switch
+
+If you have an event that teleports the player, and somehow that event also opens an BlockEntity inventory, the server will lock up waiting for chunks on another world.
+
+Example:
+```kotlin
+@EventHandler
+fun onInteract(e: PlayerInteractEvent) {
+    e.player.teleport(Location(Bukkit.getWorld("..."), 0.0, 0.0, 0.0))
+}
+```
+
+If you right-click on a chest, the player will be teleported, the chest will open... and the server will lock up!
+
+This happens because the player is teleported BEFORE the inventory has been opened, the inventory is only opened AFTER the player has been teleported.
+
+In a normal Paper server, this ain't a huge deal: The inventory is closed when the player is ticked in the new world, since it will fail the `stillValid` check.
+
+In a parallel environment however, the `stillValid` and plugins listening for `InventoryCloseEvent` while loading the `holder` may load chunks from other worlds in a different ServerLevel Tick Thread! This locks up since we CANNOT load chunks from other worlds, because well, you know, it attempts to load on the main thread, but because the main thread is blocked...
+
+To work around this issue...
+* `openMenu` will ignore any open menu requests until player `hasTickedAtLeastOnceInNewWorld`.
+* There are additional checks in the `BaseContainerBlockEntity#canUnlock`, to reject containers that were attempted to be opened after the player switched worlds.
+
+In the future, it would be nice if `openMenu` could check if the container is still valid in the new world and then decide if the container should be closed. Currently, to open inventories after a teleport, wait 1 tick.
+
+## TickThread Checks in NMS Level `setBlockEntity` and `getBlockEntity`
+
+I attempted to add TickThread checks to these two methods, however, it seems like StarLight DOES access these block entities in a different "Tuinity Chunk System Worker" thread. Heck, even CoreProtect accesses these block entities from an async thread!
+
+I looked up what Folia does, and it seems that they do have thread checks there, but it seems that they only check if it is a tick thread instead of checking if the current thread is related to the current world.
+
+To be honest, it seems that `getBlockEntity` is thread safe. EXCEPT if you are accessing block entities from a separate `ServerLevelTickThread`! This will cause a main thread chunk load, and that will freeze the server.
+
+The `capturedTileEntities` is also sus, but the map itself doesn't seem to be ever iterated, the only time it is iterated is via `entrySet()`. Maybe just to be extra sure, synchronize it? (Folia doesn't do that)
+
+So, instead of doing what Folia does, we check `getBlockEntity` it via `ensureTickThreadOrAsyncThread`.
+
+However, `setBlockEntity` still has the `ensureTickThread` check.

PARALLEL_WORLD_TICKING.md

@@ -0,0 +1,101 @@
+# Parallel World Ticking
+
+"mom can we have folia?"
+
+"we already have folia at home"
+
+folia at home: *this*
+
+## ⚠️⚠️⚠️ THIS IS EXPERIMENTAL ⚠️⚠️⚠️
+
+DON'T COMPLAIN IF YOUR SERVER EXPLODES. AFTER ALL, SPARKLYPAPER WAS MADE FOR SPARKLYPOWER, ABSOUTELY NO SUPPORT FOR YOU!!!
+
+In the Minecraft server world, there are various ways of implementing concurrent ticking, such as...
+
+* The Vanilla Way™: All worlds are ticked sequentially.
+* Parallel World Ticking: All worlds are ticked in parallel, but each tick waits for all worlds to be processed before proceeding.
+* Asynchronous World Ticking: All worlds are ticked asynchronously, each world with its own tick rate. 
+* Asynchronous Region Ticking: Chunks are split up in regions, and each region are ticked asynchronously. This is what [Folia](https://github.com/PaperMC/Folia) does.
+* Truly independent servers: Each server has its own world, so each world is completely isolated from each other.
+
+SparklyPaper moves world ticking to its first logical step after The Vanilla Way™, moving Vanilla's world ticking into separate threads, allowing worlds to be ticked in parallel. Every tick waits until all worlds finishes ticking. This means that your TPS is _mostly_ based off the MSPT of your heaviest world. Useful to spread out players to multiple worlds! (Example: Multiple Survival worlds)
+
+We do run Parallel World Ticking in production @ [SparklyPower](https://sparklypower.net/). Our server was being bottlenecked by all the things our Survival world needed to tick (such as Villagers, blocks like farmland, those pesky Axolotls, etc) that we needed a solution. We first tried to go with the "one server per world" but after looking at so much complexity we would need to handle, such as...
+
+* How are you going to do Inventory syncing?
+* How are you going to maintain multiple servers?
+* How are you going to query how many players are connected to *all* servers?
+* How are you going to implement player name autocomplete, if players may be on different servers?
+* Are you ready to reimplement vanilla commands such as `/tp` and `/give`?
+ 
+That we decided that maybe it was time to pull off a crazy patch to do parallel world ticking instead.
+
+Synchronization issues *are expected to happen*. Thread checks are still present and only the `ServerLevelTickThread` that is bound to the modified world, or the main `TickThread`, can modify world data. Plugins will break with Parallel World Ticking if they attempt to modify other worlds in a thread that isn't theirs! Plugins can work around this by scheduling a main thread task.
+
+## Well, if Folia has a superior ticking system, why not use Folia?
+
+Folia is amazing, in fact, a lot of the code used for parallel world ticking was heavily inspired by... and, uh, copied from... Folia.
+
+However, due to the way Folia works, a lot of plugins *will* break and require updates from their maintainers to make them work in Folia. Which is why Folia, by default, does not allow plugins not marked as Folia compatible to work.
+
+With Parallel World Ticking, not a lot of plugins *should* break since plugins mostly do stuff on the same world that the event has been triggered, and player actions and a bunch of other stuff are still processed on the main thread, so a lot of plugins should, hopefully, work out of the box. Of course, the downside is that Parallel World Ticking does not provide all the performance advantages that Folia has, and you are forced to break down your players into multiple worlds to get those sweet TPS improvements, while with Folia you only need to get players to be far from each other in the same world.
+
+However, there are things that WILL BREAK, such as teleporting players/entities to another world on events called on the server level tick thread. You can work around these issues by scheduling these API calls to be run after all worlds are ticked.
+
+Because Minecraft's vanilla mechanics do not interfer with other worlds, aside from portal/end portal respawn, maintaining the vanilla behavior for items is easier compared to Folia.
+
+So this is a stopgap solution while Folia isn't ready for prime time yet, without requiring you to do the whole "one servers for each world" approach, which is way harder to develop, handle, and maintain.
+
+Besides, it is fun!
+
+## If this is possible, why Paper doesn't have it built-in?
+
+Plugins, CraftBukkit, and the Minecraft Server itself, weren't really made with parallel world ticking in mind.
+
+Adding this to Paper would ensure that a lot of angry users would complain to Paper that plugin xyz isn't working. This is also the reason about why Folia only allows loading plugins marked as Folia compatible.
+
+## I've heard that anything async related in the Minecraft code is bad
+
+Yes, attempts to do ✨ async magic ✨ in the Minecraft server aren't a new thing. This has been done in the past in Akarin, Yatopia, and other forks ([patch](https://github.com/YatopiaMC/Yatopia/blob/1a54ef2f995f049d4fcf1f2bd084691126f10046/patches/server/0046-Option-for-async-world-ticking.patch)).
+
+However, the previous attempts were based on "`synchronize` and pray", which is why they were unstable and not recommended for production.
+
+MrPowerGamerBR from 2018 had even made a meme making fun of these patches.
+
+![https://cdn.discordapp.com/attachments/289587909051416579/482922902283485184/async_forks.png?ex=6558cd80&is=65465880&hm=28743988187da5dfa050c417ca9fa575c6924b6631c549f93e3186522a376c82&](https://cdn.discordapp.com/attachments/289587909051416579/482922902283485184/async_forks.png?ex=6558cd80&is=65465880&hm=28743988187da5dfa050c417ca9fa575c6924b6631c549f93e3186522a376c82&)
+
+SparklyPaper follows Folia's footsteps and keeps everything in check, keeping all tick thread checks in the code. Most of the groundwork had already been done by Spottedleaf and the Paper team. thx leaf *pets the leaf*
+
+But of course, that doesn't mean that SparklyPaper is perfect! If your server crashes, that ain't gonna be my fault xd.
+
+## I live on the edge and I don't want random "not on main thread" throws
+
+Off-main thread throws can be disabled with `-Dsparklypaper.disableHardThrow=true`, but THIS IS NOT RECOMMENDED because you SHOULD FIX THE ISSUES THEMSELVES instead of RISKING DATA CORRUPTION!!! The server will still log the stacktrace of where the exception happened.
+
+In fact, disabling throws is not an easy way out: Yes, you avoid some functions borking out. But, if the tick thread check has failed, your server is probably going to crash anyway. Example: If a plugin is attempting to teleport a player to world X while they are in a TickThread of world Y, the server will lock up because loading chunks outside of the world's tick thread or from an async thread is not allowed. In fact, if you had kept hard throws enabled, your server wouldn't have crashed because the request would've been denied! Fix the dang issues instead!!!
+
+## Profiling with Spark
+
+By default, Spark will profile the `Server thread` thread, which ain't good for us if we want to profile what is being used in our worlds.
+
+Spark has an undocumented configuration setting to configure what threads the background profiler will track.
+
+In Spark's `config.json`, add `"backgroundProfilerThreadDumper": "Server thread,serverlevel-tick-worker-1,serverlevel-tick-worker-2,serverlevel-tick-worker-3,serverlevel-tick-worker-4,serverlevel-tick-worker-5,serverlevel-tick-worker-6,serverlevel-tick-worker-7,serverlevel-tick-worker-8"` (the thread list may vary if you changed your thread count) to dump the Server thread and the ServerLevel ticking worker threads.
+
+Because Spark queries the thread list on startup, we prestart all the threads in the thread pool with `Util.SERVERLEVEL_TICK_EXECUTOR.prestartAllCoreThreads()`.
+
+## Can I disable this?
+
+SparklyPaper was tailor-made for SparklyPower with features that we need, so... no.
+
+I'm not even sure why this question is even here considering that the only real SparklyPaper user is myself. :3
+
+*Theorically* if you really want to, you can set `parallel-world-ticking.threads` to 1, and it would work just like the good old days. None of the additional checks are removed, however.
+
+## Plugin Incompatibilities
+
+[Here's a list of plugins that have issues with parallel world ticking](PARALLEL_INCOMPATIBLE_PLUGINS.md)
+
+## Implementation Notes
+
+If you are curious about things that I've learned while making this, I wrote [some notes about why some things were implemented the way that they are](PARALLEL_NOTES.md). 

README.md

@@ -10,7 +10,15 @@ SparklyPower's Paper fork, with a mix of weird & crazy patches from other forks!
 
 While our fork is mostly cherry-picked patches from other forks, we do have some handmade patches too to add and optimize some of the things that we have in our server!
 
+## Features
+
+SparklyPaper's config file is `sparklypaper.yml`, the file is, by default, placed on the root of your server.
+
 * Configurable Farm Land moisture tick rate when the block is already moisturised
-  * The isNearWater check is costly, especially if you have a lot of farm lands. If the block is already moistured, we can change the tick rate of it to avoid these expensive isNearWater checks.
+  * The `isNearWater` check is costly, especially if you have a lot of farm lands. If the block is already moistured, we can change the tick rate of it to avoid these expensive `isNearWater` checks.
+* Check how much MSPT (milliseconds per tick) each world is using in `/mspt`
+  * Useful to figure out which worlds are lagging your server.
+* Parallel World Ticking
+  * "mom can we have folia?" "we already have folia at home" folia at home: [Parallel World Ticking](PARALLEL_WORLD_TICKING.md)
 
 We don't cherry-pick *everything* from other forks, only patches that I can see and think "yeah, I can see how this would improve performance" or patches that target specific performance/feature pain points in our server are cherry-picked! In fact, some patches that are used in other forks [may be actually borked](BORKED_PATCHES.md)...

gradle.properties

@@ -2,7 +2,7 @@ group=net.sparklypower.sparklypaper
 version=1.20.2-R0.1-SNAPSHOT
 
 mcVersion=1.20.2
-paperRef=4675152f4908431e0f944a7bf9fa3b2181a2dfd6
+paperRef=f186318a91cbd3b2a2259d39cb88576989a496b8
 
 org.gradle.caching=true
 org.gradle.parallel=true

patches/server/0001-new-fork-who-dis-Rebrand-to-SparklyPaper-and-Build-C.patch

@@ -5,7 +5,7 @@ Subject: [PATCH] new fork who dis - Rebrand to SparklyPaper and Build Changes
 
 
 diff --git a/build.gradle.kts b/build.gradle.kts
-index a79461457ea19339f47572c70705d655ebc55276..d43c3f2e17f9ef48ec458f9c94478cfd02db6edb 100644
+index 79beac737c17412913983614bd478d33e3c6ed58..8adfd75d66cbd3e3afafc0ea167d1e6568c4adbe 100644
 --- a/build.gradle.kts
 +++ b/build.gradle.kts
 @@ -3,6 +3,8 @@ import io.papermc.paperweight.util.*
@@ -54,7 +54,7 @@ index a79461457ea19339f47572c70705d655ebc55276..d43c3f2e17f9ef48ec458f9c94478cfd
      standardInput = System.`in`
      workingDir = rootProject.layout.projectDirectory
 diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
-index 97745f0bab8d82d397c6c2a5775aed92bca0a034..0dfd9a2f9195ec018ed5069f43908b8c0e09edbd 100644
+index 8f31413c939cc2b0454ad3d9a1b618dbae449d00..25367df06a8a6e8b0b3a56652a5fb1c70a15632d 100644
 --- a/src/main/java/net/minecraft/server/MinecraftServer.java
 +++ b/src/main/java/net/minecraft/server/MinecraftServer.java
 @@ -1697,7 +1697,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
@@ -67,10 +67,10 @@ index 97745f0bab8d82d397c6c2a5775aed92bca0a034..0dfd9a2f9195ec018ed5069f43908b8c
  
      public SystemReport fillSystemReport(SystemReport details) {
 diff --git a/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
-index b7e7e6ed60f55d2ab5e4fcefb3638ad1768c3b7f..38460bef2c55264b145c3e21b314eeb88b812d41 100644
+index 9c08303de2891de92e06de8a939a618b7a6f7321..aaf04e2be3ed707e51a81d2b1c58dda6f7a8092a 100644
 --- a/src/main/java/org/bukkit/craftbukkit/CraftServer.java
 +++ b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
-@@ -267,7 +267,7 @@ import javax.annotation.Nullable; // Paper
+@@ -269,7 +269,7 @@ import javax.annotation.Nullable; // Paper
  import javax.annotation.Nonnull; // Paper
  
  public final class CraftServer implements Server {
@@ -93,7 +93,7 @@ index 774556a62eb240da42e84db4502e2ed43495be17..22e504565de83f976c8b2996f0b2207b
  
          if (stream != null) {
 diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java
-index 50c72e5db369a180f425eaaa0411cb8871bc3463..682077870a3607afc7ec66ed6e8dcf8484c63fd0 100644
+index 40dcdf6885e99b26283a9ea2bd4d4bf6ec358e71..d502b0d45bc950a586049dd5cd242319da5ba721 100644
 --- a/src/main/java/org/spigotmc/WatchdogThread.java
 +++ b/src/main/java/org/spigotmc/WatchdogThread.java
 @@ -155,14 +155,14 @@ public final class WatchdogThread extends io.papermc.paper.util.TickThread // Pa

patches/server/0002-SparklyPaper-config-files.patch

@@ -5,7 +5,7 @@ Subject: [PATCH] SparklyPaper config files
 
 
 diff --git a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
-index 2b5d82fbf4e7ec32d0c53dd3e8207b1dba708bbd..c67f33a21f0061ff910653b6a911648967145d19 100644
+index cf605aa56adf7f80d3b409f60a92a5ca7ae8fd07..8c20221736419bcb9c3e570af624eef8e6fc3b09 100644
 --- a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
 +++ b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
 @@ -218,6 +218,15 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface
@@ -116,7 +116,7 @@ index 0000000000000000000000000000000000000000..bc0ec96f91f7c9ab9f9a865a50f69707
 +    }
 +}
 diff --git a/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
-index 8c5bf002fb437d192132ef90324a095dd064fea8..9dad8611d887e80207385c36c8983104903d092d 100644
+index aaf04e2be3ed707e51a81d2b1c58dda6f7a8092a..bcc98faf17782c3d7ba576b7d605446a9a4d864b 100644
 --- a/src/main/java/org/bukkit/craftbukkit/CraftServer.java
 +++ b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
 @@ -117,6 +117,7 @@ import net.minecraft.world.level.storage.WorldData;
@@ -127,7 +127,7 @@ index 8c5bf002fb437d192132ef90324a095dd064fea8..9dad8611d887e80207385c36c8983104
  import org.bukkit.BanList;
  import org.bukkit.Bukkit;
  import org.bukkit.ChatColor;
-@@ -1039,6 +1040,7 @@ public final class CraftServer implements Server {
+@@ -1041,6 +1042,7 @@ public final class CraftServer implements Server {
  
          org.spigotmc.SpigotConfig.init((File) this.console.options.valueOf("spigot-settings")); // Spigot
          this.console.paperConfigurations.reloadConfigs(this.console);
@@ -135,7 +135,7 @@ index 8c5bf002fb437d192132ef90324a095dd064fea8..9dad8611d887e80207385c36c8983104
          for (ServerLevel world : this.console.getAllLevels()) {
              // world.serverLevelData.setDifficulty(config.difficulty); // Paper - per level difficulty
              world.setSpawnSettings(world.serverLevelData.getDifficulty() != Difficulty.PEACEFUL && config.spawnMonsters, config.spawnAnimals); // Paper - per level difficulty (from MinecraftServer#setDifficulty(ServerLevel, Difficulty, boolean))
-@@ -1054,6 +1056,7 @@ public final class CraftServer implements Server {
+@@ -1056,6 +1058,7 @@ public final class CraftServer implements Server {
                  }
              }
              world.spigotConfig.init(); // Spigot
@@ -143,7 +143,7 @@ index 8c5bf002fb437d192132ef90324a095dd064fea8..9dad8611d887e80207385c36c8983104
          }
  
          Plugin[] pluginClone = pluginManager.getPlugins().clone(); // Paper
-@@ -1069,6 +1072,7 @@ public final class CraftServer implements Server {
+@@ -1071,6 +1074,7 @@ public final class CraftServer implements Server {
          this.reloadData();
          org.spigotmc.SpigotConfig.registerCommands(); // Spigot
          io.papermc.paper.command.PaperCommands.registerCommands(this.console); // Paper
@@ -201,10 +201,10 @@ index 0000000000000000000000000000000000000000..614e64ce6bf5bb7fab5758250927a0d3
 \ No newline at end of file
 diff --git a/src/main/kotlin/net/sparklypower/sparklypaper/configs/SparklyPaperConfig.kt b/src/main/kotlin/net/sparklypower/sparklypaper/configs/SparklyPaperConfig.kt
 new file mode 100644
-index 0000000000000000000000000000000000000000..1965b785204edcb8199bfe8b7ab1c86892c848c0
+index 0000000000000000000000000000000000000000..6398c7b40ba82ffc8588eca458ce92c24dc8ac41
 --- /dev/null
 +++ b/src/main/kotlin/net/sparklypower/sparklypaper/configs/SparklyPaperConfig.kt
-@@ -0,0 +1,10 @@
+@@ -0,0 +1,17 @@
 +package net.sparklypower.sparklypaper.configs
 +
 +import kotlinx.serialization.SerialName
@@ -212,16 +212,23 @@ index 0000000000000000000000000000000000000000..1965b785204edcb8199bfe8b7ab1c868
 +
 +@Serializable
 +class SparklyPaperConfig(
++    @SerialName("parallel-world-ticking")
++    val parallelWorldTicking: ParallelWorldTicking,
 +    @SerialName("world-settings")
 +    val worldSettings: Map<String, SparklyPaperWorldConfig>
-+)
++) {
++    @Serializable
++    class ParallelWorldTicking(
++        val threads: Int
++    )
++}
 \ No newline at end of file
 diff --git a/src/main/kotlin/net/sparklypower/sparklypaper/configs/SparklyPaperConfigUtils.kt b/src/main/kotlin/net/sparklypower/sparklypaper/configs/SparklyPaperConfigUtils.kt
 new file mode 100644
-index 0000000000000000000000000000000000000000..82a29b23429e31d78e09fa23e8c87cec76ba63bd
+index 0000000000000000000000000000000000000000..fbbb11c1a62a28a251c35261fb29e6267a08c1a3
 --- /dev/null
 +++ b/src/main/kotlin/net/sparklypower/sparklypaper/configs/SparklyPaperConfigUtils.kt
-@@ -0,0 +1,46 @@
+@@ -0,0 +1,49 @@
 +package net.sparklypower.sparklypaper.configs
 +
 +import com.charleskorn.kaml.Yaml
@@ -243,6 +250,9 @@ index 0000000000000000000000000000000000000000..82a29b23429e31d78e09fa23e8c87cec
 +            configFile.writeText(
 +                yaml.encodeToString(
 +                    SparklyPaperConfig(
++                        SparklyPaperConfig.ParallelWorldTicking(
++                            threads = 8
++                        ),
 +                        mapOf(
 +                            "default" to SparklyPaperWorldConfig(
 +                                SparklyPaperWorldConfig.TickRates(