275 lines
12 KiB
Diff
275 lines
12 KiB
Diff
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
|
From: MrHua269 <novau233@163.com>
|
|
Date: Sun, 10 Mar 2024 05:04:33 +0000
|
|
Subject: [PATCH] Add a simple watchdog for tick regions
|
|
|
|
|
|
diff --git a/src/main/java/io/papermc/paper/threadedregions/RegionizedServer.java b/src/main/java/io/papermc/paper/threadedregions/RegionizedServer.java
|
|
index fc3d332aa1c1d469cedfe2aaa7102dcd78e25642..1ae61bc6603dd3ac290e3ead20416f9c5b63ff02 100644
|
|
--- a/src/main/java/io/papermc/paper/threadedregions/RegionizedServer.java
|
|
+++ b/src/main/java/io/papermc/paper/threadedregions/RegionizedServer.java
|
|
@@ -279,6 +279,7 @@ public final class RegionizedServer {
|
|
*/
|
|
|
|
private void globalTick(final int tickCount) {
|
|
+ me.earthme.luminol.utils.LuminolWatchDog.globalRegionHeartBeat(); //Luminol
|
|
/*
|
|
if (false) {
|
|
io.papermc.paper.threadedregions.ThreadedTicketLevelPropagator.main(null);
|
|
diff --git a/src/main/java/io/papermc/paper/threadedregions/TickRegions.java b/src/main/java/io/papermc/paper/threadedregions/TickRegions.java
|
|
index 2ad25dd345ab42125d456f2b9cf67d8c4515c8b7..183fede965c3d227bbcc7e54781869e09796081f 100644
|
|
--- a/src/main/java/io/papermc/paper/threadedregions/TickRegions.java
|
|
+++ b/src/main/java/io/papermc/paper/threadedregions/TickRegions.java
|
|
@@ -402,10 +402,11 @@ public final class TickRegions implements ThreadedRegionizer.RegionCallbacks<Tic
|
|
|
|
@Override
|
|
protected void tickRegion(final int tickCount, final long startTime, final long scheduledEnd) {
|
|
+ me.earthme.luminol.utils.LuminolWatchDog.tickRegionHeartBeat(); //Luminol - Simple watchdog
|
|
final ca.spottedleaf.leafprofiler.RegionizedProfiler.Handle profiler = io.papermc.paper.threadedregions.TickRegionScheduler.getProfiler(); // Folia - profiler
|
|
profiler.startTick(); try { // Folia - profiler
|
|
MinecraftServer.getServer().tickServer(startTime, scheduledEnd, TimeUnit.MILLISECONDS.toMillis(10L), this.region);
|
|
- } finally { profiler.stopTick(); } // Folia - profiler
|
|
+ } finally { profiler.stopTick(); me.earthme.luminol.utils.LuminolWatchDog.releaseTickRegion(); } // Folia - profiler //Luminol - Simple watchdog
|
|
}
|
|
|
|
@Override
|
|
diff --git a/src/main/java/me/earthme/luminol/config/modules/misc/WatchdogConfig.java b/src/main/java/me/earthme/luminol/config/modules/misc/WatchdogConfig.java
|
|
new file mode 100644
|
|
index 0000000000000000000000000000000000000000..71d80b401470db777a86274d32f178bb2aa023d5
|
|
--- /dev/null
|
|
+++ b/src/main/java/me/earthme/luminol/config/modules/misc/WatchdogConfig.java
|
|
@@ -0,0 +1,27 @@
|
|
+package me.earthme.luminol.config.modules.misc;
|
|
+
|
|
+import me.earthme.luminol.config.ConfigInfo;
|
|
+import me.earthme.luminol.config.EnumConfigCategory;
|
|
+import me.earthme.luminol.config.HotReloadUnsupported;
|
|
+import me.earthme.luminol.config.IConfigModule;
|
|
+
|
|
+public class WatchdogConfig implements IConfigModule {
|
|
+
|
|
+ @HotReloadUnsupported
|
|
+ @ConfigInfo(baseName = "enabled")
|
|
+ public static boolean enabled = false;
|
|
+ @ConfigInfo(baseName = "warn_period_ticks")
|
|
+ public static long warnPeriodTicks = 5 * 20;
|
|
+ @ConfigInfo(baseName = "timeout_ticks")
|
|
+ public static long timeOutTicks = 30 * 20;
|
|
+
|
|
+ @Override
|
|
+ public EnumConfigCategory getCategory() {
|
|
+ return EnumConfigCategory.MISC;
|
|
+ }
|
|
+
|
|
+ @Override
|
|
+ public String getBaseName() {
|
|
+ return "watchdog";
|
|
+ }
|
|
+}
|
|
diff --git a/src/main/java/me/earthme/luminol/utils/LuminolWatchDog.java b/src/main/java/me/earthme/luminol/utils/LuminolWatchDog.java
|
|
new file mode 100644
|
|
index 0000000000000000000000000000000000000000..e19c6330a48cee402b1a999ca71cb2329d530356
|
|
--- /dev/null
|
|
+++ b/src/main/java/me/earthme/luminol/utils/LuminolWatchDog.java
|
|
@@ -0,0 +1,178 @@
|
|
+package me.earthme.luminol.utils;
|
|
+
|
|
+import com.google.common.collect.Maps;
|
|
+import io.papermc.paper.threadedregions.ThreadedRegionizer;
|
|
+import io.papermc.paper.threadedregions.TickRegionScheduler;
|
|
+import io.papermc.paper.util.StacktraceDeobfuscator;
|
|
+import me.earthme.luminol.config.modules.misc.WatchdogConfig;
|
|
+import net.minecraft.server.MinecraftServer;
|
|
+import org.apache.commons.lang3.tuple.Pair;
|
|
+import org.jetbrains.annotations.NotNull;
|
|
+import org.jetbrains.annotations.Nullable;
|
|
+import org.slf4j.Logger;
|
|
+
|
|
+import java.lang.management.ManagementFactory;
|
|
+import java.lang.management.ThreadInfo;
|
|
+import java.util.ArrayList;
|
|
+import java.util.List;
|
|
+import java.util.Map;
|
|
+import java.util.concurrent.CompletableFuture;
|
|
+import java.util.concurrent.Executor;
|
|
+import java.util.concurrent.Executors;
|
|
+import java.util.concurrent.TimeUnit;
|
|
+
|
|
+public class LuminolWatchDog {
|
|
+ private static final Object keepaliveDataLock = new Object();
|
|
+ private static Pair<Thread,Long> globalRegionLastKeepalive = null;
|
|
+ private static final Map<Thread,Pair<ThreadedRegionizer.ThreadedRegion<?,?>,Long>> otherTickRegionKeepalive = Maps.newHashMap();
|
|
+ private static final Executor checkTimer = CompletableFuture.delayedExecutor(50, TimeUnit.MILLISECONDS,Executors.newSingleThreadExecutor());
|
|
+ private static final Logger logger = MinecraftServer.LOGGER;
|
|
+ private static int tickCount = 0;
|
|
+ private static volatile boolean runScheduleNext = true;
|
|
+
|
|
+ public static void boot(){
|
|
+ if (!WatchdogConfig.enabled){
|
|
+ return;
|
|
+ }
|
|
+
|
|
+ runCheck();
|
|
+ }
|
|
+
|
|
+ private static void scheduleCheck(){
|
|
+ checkTimer.execute(LuminolWatchDog::runCheck);
|
|
+ }
|
|
+
|
|
+ public static void stopTicking(){
|
|
+ runScheduleNext = false;
|
|
+ }
|
|
+
|
|
+ private static void runCheck(){
|
|
+ try {
|
|
+ if (MinecraftServer.getServer().isStopped() || !runScheduleNext){
|
|
+ return;
|
|
+ }
|
|
+
|
|
+ tickCount++;
|
|
+
|
|
+ final List<Pair<Thread,Long>> threadsToWarn = new ArrayList<>();
|
|
+ final long currentTimeNano = System.nanoTime();
|
|
+ boolean shouldStopServer = false;
|
|
+
|
|
+ synchronized (keepaliveDataLock){
|
|
+ if (globalRegionLastKeepalive != null){
|
|
+ final Thread lastGlobalRegionThread = globalRegionLastKeepalive.getLeft();
|
|
+ final long lastTicked = globalRegionLastKeepalive.getRight();
|
|
+
|
|
+ final long timeEscaped = currentTimeNano - lastTicked;
|
|
+
|
|
+ if (timeEscaped >= WatchdogConfig.warnPeriodTicks * 50 * 1000 * 1000){
|
|
+ threadsToWarn.add(Pair.of(lastGlobalRegionThread,timeEscaped));
|
|
+ }
|
|
+ }
|
|
+
|
|
+ for (Map.Entry<Thread,Pair<ThreadedRegionizer.ThreadedRegion<?,?>,Long>> tickData : otherTickRegionKeepalive.entrySet()){
|
|
+ final Thread targetThread = tickData.getKey();
|
|
+ final Pair<ThreadedRegionizer.ThreadedRegion<?, ?>, Long> keepaliveData = tickData.getValue();
|
|
+ final long lastKeepalive = keepaliveData.getRight();
|
|
+
|
|
+ final long timeEscaped = currentTimeNano - lastKeepalive;
|
|
+
|
|
+ if (timeEscaped >= WatchdogConfig.warnPeriodTicks * 50 * 1000 * 1000){
|
|
+ threadsToWarn.add(Pair.of(targetThread,timeEscaped));
|
|
+ }
|
|
+ }
|
|
+
|
|
+ for(Pair<Thread,Long> warnInfo : threadsToWarn){
|
|
+ final Thread targetThread = warnInfo.getLeft();
|
|
+ final long timeEscaped = warnInfo.getRight();
|
|
+
|
|
+ final ThreadedRegionizer.ThreadedRegion<?,?> targetRegion = targetThread == globalRegionLastKeepalive.getKey() ? null : otherTickRegionKeepalive.get(targetThread).getLeft();
|
|
+
|
|
+ if (tickCount % WatchdogConfig.warnPeriodTicks == 0){
|
|
+ dumpSingleRegion(targetThread,targetRegion,timeEscaped);
|
|
+ }
|
|
+
|
|
+ if (timeEscaped > WatchdogConfig.timeOutTicks * 50 * 1000 * 1000){
|
|
+ shouldStopServer = true;
|
|
+ runScheduleNext = false;
|
|
+ }
|
|
+ }
|
|
+
|
|
+ if (shouldStopServer){
|
|
+ MinecraftServer.getServer().stopServer();
|
|
+ }
|
|
+ }
|
|
+ }finally {
|
|
+ if (runScheduleNext){
|
|
+ scheduleCheck();
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+
|
|
+ private static void dumpSingleRegion(@NotNull Thread thread, @Nullable ThreadedRegionizer.ThreadedRegion<?,?> region,long timeEscaped){
|
|
+ final StringBuilder messageBuilder = new StringBuilder();
|
|
+ messageBuilder.append("====================================").append("\n");
|
|
+ messageBuilder.append("There is a single region that has no responding in " + TimeUnit.NANOSECONDS.toSeconds(timeEscaped) + " seconds!").append("\n");
|
|
+ messageBuilder.append("If you believe this is a bug,please send it to github issue after checking your plugins!").append("\n");
|
|
+ messageBuilder.append("Tick Region: ").append(region == null ? "Global Region" : "ChunkX(Center): " + region.getCenterChunk().x + " ChunkZ(Center): " + region.getCenterChunk().z).append("\n");
|
|
+ messageBuilder.append("Thread Name: ").append(thread.getName()).append("\n");
|
|
+ messageBuilder.append("Thread Id: ").append(thread.getId()).append("\n");
|
|
+
|
|
+ final ThreadInfo targetThreadInfo = ManagementFactory.getThreadMXBean().getThreadInfo(thread.getId());
|
|
+
|
|
+ messageBuilder.append("Is Suspend: ").append(targetThreadInfo.isSuspended()).append("\n");
|
|
+ messageBuilder.append("Thread State: ").append(targetThreadInfo.getThreadState()).append("\n");
|
|
+ messageBuilder.append("Thread Priority: ").append(targetThreadInfo.getPriority()).append("\n");
|
|
+ messageBuilder.append("Thread Stack Trace:").append("\n");
|
|
+
|
|
+ for (StackTraceElement stackTraceElement : StacktraceDeobfuscator.INSTANCE.deobfuscateStacktrace(thread.getStackTrace())){
|
|
+ messageBuilder.append("\t\t").append(stackTraceElement).append("\n");
|
|
+ }
|
|
+
|
|
+ messageBuilder.append("\n");
|
|
+ messageBuilder.append("====================================");
|
|
+
|
|
+ logger.error(messageBuilder.toString());
|
|
+ }
|
|
+
|
|
+ public static void globalRegionHeartBeat(){
|
|
+ if (!WatchdogConfig.enabled){
|
|
+ return;
|
|
+ }
|
|
+
|
|
+ synchronized (keepaliveDataLock){
|
|
+ globalRegionLastKeepalive = Pair.of(Thread.currentThread(),System.nanoTime());
|
|
+ }
|
|
+ }
|
|
+
|
|
+ public static void tickRegionHeartBeat(){
|
|
+ if (!WatchdogConfig.enabled){
|
|
+ return;
|
|
+ }
|
|
+
|
|
+ final ThreadedRegionizer.ThreadedRegion<?,?> currentRegion = TickRegionScheduler.getCurrentRegion();
|
|
+
|
|
+ if (currentRegion == null) {
|
|
+ return;
|
|
+ }
|
|
+
|
|
+ synchronized (keepaliveDataLock){
|
|
+ if (!otherTickRegionKeepalive.containsKey(Thread.currentThread())){
|
|
+ otherTickRegionKeepalive.put(Thread.currentThread(),Pair.of(currentRegion,System.nanoTime()));
|
|
+ return;
|
|
+ }
|
|
+
|
|
+ otherTickRegionKeepalive.replace(Thread.currentThread(),Pair.of(currentRegion,System.nanoTime()));
|
|
+ }
|
|
+ }
|
|
+
|
|
+ public static void releaseTickRegion(){
|
|
+ if (!WatchdogConfig.enabled){
|
|
+ return;
|
|
+ }
|
|
+
|
|
+ synchronized (keepaliveDataLock){
|
|
+ otherTickRegionKeepalive.remove(Thread.currentThread());
|
|
+ }
|
|
+ }
|
|
+}
|
|
\ No newline at end of file
|
|
diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
|
|
index 3a6d567ee9db93606aec30255400dff4f9f87b05..14114e868d3547f2f2d4149312742d5e995e8959 100644
|
|
--- a/src/main/java/net/minecraft/server/MinecraftServer.java
|
|
+++ b/src/main/java/net/minecraft/server/MinecraftServer.java
|
|
@@ -1003,6 +1003,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
|
|
// Folia end - region threading
|
|
|
|
public void stopServer() {
|
|
+ me.earthme.luminol.utils.LuminolWatchDog.stopTicking(); // Luminol - Watchdog
|
|
// Folia start - region threading
|
|
// halt scheduler
|
|
// don't wait, we may be on a scheduler thread
|
|
@@ -1223,6 +1224,8 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
|
|
this.statusIcon = (ServerStatus.Favicon) this.loadStatusIcon().orElse(null); // CraftBukkit - decompile error
|
|
this.status = this.buildServerStatus();
|
|
|
|
+ me.earthme.luminol.utils.LuminolWatchDog.boot(); //Luminol - Simple watchdog
|
|
+
|
|
// Folia start - region threading
|
|
if (true) {
|
|
io.papermc.paper.threadedregions.RegionizedServer.getInstance().init(); // Folia - region threading - only after loading worlds
|