Repo-Mirror/KeYiMC
9
0
Fork 0
Code Issues Pull Requests Releases Activity

Use better executors

Browse Source
This commit is contained in:
nostalgic853
2022-12-02 12:02:42 +08:00
parent 4032c984ce
commit bfb566a091
8 changed files with 427 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
patches/server/0023-Add-centralized-AsyncExecutor.patch Normal file
View File

@@ -0,0 +1,117 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: MartijnMuijsers <martijnmuijsers@live.nl>
Date: Tue, 29 Nov 2022 12:35:35 +0100
Subject: [PATCH] Add centralized AsyncExecutor
./gs://www.gnu.org/licenses/agpl-3.0.html)
This patch was taken from Gale.
diff --git a/src/main/java/org/galemc/gale/concurrent/AsyncExecutor.java b/src/main/java/org/galemc/gale/concurrent/AsyncExecutor.java
new file mode 100644
index 0000000000000000000000000000000000000000..de182b9473963b95085fa612f70884a56765ae43
--- /dev/null
+++ b/src/main/java/org/galemc/gale/concurrent/AsyncExecutor.java
@@ -0,0 +1,103 @@
+// Gale - centralized async execution
+
+package org.galemc.gale.concurrent;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import net.minecraft.Util;
+import org.galemc.gale.util.CPUCoresEstimation;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * An executor for tasks that can run asynchronously. This executor uses a fixed thread pool, and as such
+ * is not appropriate for tasks that block.
+ * <br>
+ * It can be paused when all CPU cores may be needed for something else.
+ *
+ * @author Martijn Muijsers
+ */
+public final class AsyncExecutor extends ThreadPoolExecutor {
+
+ /**
+ * The fixed number of threads that will be used by this {@link AsyncExecutor}.
+ * <br>
+ * By default, we do not use two cores, so that there is always a core for the main thread that we do not use,
+ * and another core that we do not use to run other important threads such as garbage collection on.
+ * <br>
+ * This value is at least 1.
+ */
+ public static final int parallelism;
+ static {
+ int parallelismByEnvironmentVariable = Integer.getInteger("gale.threads.async", -1);
+ parallelism = Math.max(1, parallelismByEnvironmentVariable > 0 ? parallelismByEnvironmentVariable : CPUCoresEstimation.get() - 2);
+ }
+
+ /**
+ * The queue of tasks in the {@link AsyncExecutor} singleton instance.
+ * This queue can be accessed externally to steal work from the executor.
+ */
+ public static final BlockingQueue<Runnable> queue = new LinkedBlockingQueue<>();
+
+ /**
+ * Singleton {@link AsyncExecutor} instance.
+ */
+ public static final AsyncExecutor instance = new AsyncExecutor();
+
+ private static volatile boolean isPaused = false;
+ private static final ReentrantLock pauseLock = new ReentrantLock();
+ private static final Condition pauseCondition = pauseLock.newCondition();
+
+ private AsyncExecutor() {
+ super(parallelism, parallelism, 0L, TimeUnit.MILLISECONDS, queue, new ThreadFactoryBuilder()
+ .setNameFormat("Async Executor Thread - %1$d")
+ .setPriority(Thread.NORM_PRIORITY - 1) // Deprioritize over main
+ .setUncaughtExceptionHandler(Util::onThreadException)
+ .build());
+ }
+
+ @Override
+ protected void beforeExecute(Thread t, Runnable r) {
+ super.beforeExecute(t, r);
+ pauseLock.lock();
+ try {
+ while (isPaused) pauseCondition.await();
+ } catch (InterruptedException ie) {
+ t.interrupt();
+ } finally {
+ pauseLock.unlock();
+ }
+ }
+
+ /**
+ * Pauses the {@link AsyncExecutor} from starting to run any new task until {@link #resume()} is called.
+ * <br>
+ * This does not affect execution of tasks that are already being performed when this method is called.
+ */
+ public static void pause() {
+ pauseLock.lock();
+ try {
+ isPaused = true;
+ } finally {
+ pauseLock.unlock();
+ }
+ }
+
+ /**
+ * Resumes the {@link AsyncExecutor} singleton instance after it has been paused using {@link #pause()}.
+ */
+ public static void resume() {
+ pauseLock.lock();
+ try {
+ isPaused = false;
+ pauseCondition.signalAll();
+ } finally {
+ pauseLock.unlock();
+ }
+ }
+
+}

70
patches/server/0024-Remove-Paper-async-executor.patch Normal file
View File

@@ -0,0 +1,70 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: MartijnMuijsers <martijnmuijsers@live.nl>
Date: Tue, 29 Nov 2022 13:10:20 +0100
Subject: [PATCH] Remove Paper async executor
License: AGPL-3.0 (https://www.gnu.org/licenses/agpl-3.0.html)
This patch was taken from Gale.
diff --git a/src/main/java/io/papermc/paper/util/MCUtil.java b/src/main/java/io/papermc/paper/util/MCUtil.java
index b5d4c53bf1046fa52da5398491258b94f1e0fcd0..d5f6fc421bea74b711f99fc14ff916643bfe071a 100644
--- a/src/main/java/io/papermc/paper/util/MCUtil.java
+++ b/src/main/java/io/papermc/paper/util/MCUtil.java
@@ -34,6 +34,7 @@ import org.bukkit.Location;
import org.bukkit.block.BlockFace;
import org.bukkit.craftbukkit.CraftWorld;
import org.bukkit.craftbukkit.util.Waitable;
+import org.galemc.gale.concurrent.AsyncExecutor;
import org.spigotmc.AsyncCatcher;
import javax.annotation.Nonnull;
@@ -45,6 +46,7 @@ import java.util.Queue;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@@ -55,14 +57,7 @@ import java.util.function.Consumer;
import java.util.function.Supplier;
public final class MCUtil {
- public static final ThreadPoolExecutor asyncExecutor = new ThreadPoolExecutor(
- 0, 2, 60L, TimeUnit.SECONDS,
- new LinkedBlockingQueue<>(),
- new ThreadFactoryBuilder()
- .setNameFormat("Paper Async Task Handler Thread - %1$d")
- .setUncaughtExceptionHandler(new net.minecraft.DefaultUncaughtExceptionHandlerWithName(MinecraftServer.LOGGER))
- .build()
- );
+ public static final Executor asyncExecutor = AsyncExecutor.instance; // Gale - centralized async execution - remove Paper async executor
public static final ThreadPoolExecutor cleanerExecutor = new ThreadPoolExecutor(
1, 1, 0L, TimeUnit.SECONDS,
new LinkedBlockingQueue<>(),
diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
index ae95e3af64811f6f149a01b9715d8fdd2c2583fa..462689f5c35c5379a0281fe61ad91ae3288d279d 100644
--- a/src/main/java/net/minecraft/server/MinecraftServer.java
+++ b/src/main/java/net/minecraft/server/MinecraftServer.java
@@ -155,6 +155,7 @@ import net.minecraft.world.phys.Vec2;
import net.minecraft.world.phys.Vec3;
import org.apache.commons.lang3.Validate;
import org.slf4j.Logger;
+import org.galemc.gale.concurrent.AsyncExecutor; // Gale
// CraftBukkit start
import com.mojang.serialization.DynamicOps;
@@ -993,8 +994,10 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
MinecraftServer.LOGGER.error("Failed to unlock level {}", this.storageSource.getLevelId(), ioexception1);
}
// Spigot start
- io.papermc.paper.util.MCUtil.asyncExecutor.shutdown(); // Paper
- try { io.papermc.paper.util.MCUtil.asyncExecutor.awaitTermination(30, java.util.concurrent.TimeUnit.SECONDS); // Paper
+ // Gale start - centralized async execution - remove Paper async executor
+ AsyncExecutor.instance.shutdown(); // Paper
+ try { AsyncExecutor.instance.awaitTermination(30, java.util.concurrent.TimeUnit.SECONDS); // Paper
+ // Gale end - centralized async execution - remove Paper async executor
} catch (java.lang.InterruptedException ignored) {} // Paper
if (org.spigotmc.SpigotConfig.saveUserCacheOnStopOnly) {
MinecraftServer.LOGGER.info("Saving usercache.json");

29
patches/server/0025-Remove-Paper-cleaner-executor.patch Normal file
View File

@@ -0,0 +1,29 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: MartijnMuijsers <martijnmuijsers@live.nl>
Date: Tue, 29 Nov 2022 13:11:58 +0100
Subject: [PATCH] Remove Paper cleaner executor
License: AGPL-3.0 (https://www.gnu.org/licenses/agpl-3.0.html)
This patch was taken from Gale.
diff --git a/src/main/java/io/papermc/paper/util/MCUtil.java b/src/main/java/io/papermc/paper/util/MCUtil.java
index d5f6fc421bea74b711f99fc14ff916643bfe071a..c6c1b194fdee8841a7745b5ee3ef14e416df02d3 100644
--- a/src/main/java/io/papermc/paper/util/MCUtil.java
+++ b/src/main/java/io/papermc/paper/util/MCUtil.java
@@ -58,14 +58,7 @@ import java.util.function.Supplier;
public final class MCUtil {
public static final Executor asyncExecutor = AsyncExecutor.instance; // Gale - centralized async execution - remove Paper async executor
- public static final ThreadPoolExecutor cleanerExecutor = new ThreadPoolExecutor(
- 1, 1, 0L, TimeUnit.SECONDS,
- new LinkedBlockingQueue<>(),
- new ThreadFactoryBuilder()
- .setNameFormat("Paper Object Cleaner")
- .setUncaughtExceptionHandler(new net.minecraft.DefaultUncaughtExceptionHandlerWithName(MinecraftServer.LOGGER))
- .build()
- );
+ public static final Executor cleanerExecutor = AsyncExecutor.instance; // Gale - centralized async execution - remove Paper cleaner executor
public static final long INVALID_CHUNK_KEY = getCoordinateKey(Integer.MAX_VALUE, Integer.MAX_VALUE);

80
patches/server/0026-Remove-background-executor.patch Normal file
View File

@@ -0,0 +1,80 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: MartijnMuijsers <martijnmuijsers@live.nl>
Date: Tue, 29 Nov 2022 14:21:44 +0100
Subject: [PATCH] Remove background executor
License: AGPL-3.0 (https://www.gnu.org/licenses/agpl-3.0.html)
This patch was taken from Gale.
diff --git a/src/main/java/net/minecraft/Util.java b/src/main/java/net/minecraft/Util.java
index 6b7943e8348b0a41ca69fb56ccfd5f1c1484eb07..e14245a77b40fca4bacf82295ad390339aba08a9 100644
--- a/src/main/java/net/minecraft/Util.java
+++ b/src/main/java/net/minecraft/Util.java
@@ -72,6 +72,7 @@ import net.minecraft.util.RandomSource;
import net.minecraft.util.TimeSource;
import net.minecraft.util.datafix.DataFixers;
import net.minecraft.world.level.block.state.properties.Property;
+import org.galemc.gale.concurrent.AsyncExecutor;
import org.slf4j.Logger;
public class Util {
@@ -80,7 +81,7 @@ public class Util {
private static final String MAX_THREADS_SYSTEM_PROPERTY = "max.bg.threads";
private static final AtomicInteger WORKER_COUNT = new AtomicInteger(1);
private static final ExecutorService BOOTSTRAP_EXECUTOR = makeExecutor("Bootstrap", -2); // Paper - add -2 priority
- private static final ExecutorService BACKGROUND_EXECUTOR = makeExecutor("Main", -1); // Paper - add -1 priority
+ private static final ExecutorService BACKGROUND_EXECUTOR = AsyncExecutor.instance; // Gale - centralized async execution - remove background executor
// Paper start - don't submit BLOCKING PROFILE LOOKUPS to the world gen thread
public static final ExecutorService PROFILE_EXECUTOR = Executors.newFixedThreadPool(2, new java.util.concurrent.ThreadFactory() {
@@ -219,7 +220,6 @@ public class Util {
}
public static void shutdownExecutors() {
- shutdownExecutor(BACKGROUND_EXECUTOR);
shutdownExecutor(IO_POOL);
}
diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
index 462689f5c35c5379a0281fe61ad91ae3288d279d..b77b1bc9a5ce1373ff5c5aacc3f4c68c1c735849 100644
--- a/src/main/java/net/minecraft/server/MinecraftServer.java
+++ b/src/main/java/net/minecraft/server/MinecraftServer.java
@@ -994,11 +994,6 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
MinecraftServer.LOGGER.error("Failed to unlock level {}", this.storageSource.getLevelId(), ioexception1);
}
// Spigot start
- // Gale start - centralized async execution - remove Paper async executor
- AsyncExecutor.instance.shutdown(); // Paper
- try { AsyncExecutor.instance.awaitTermination(30, java.util.concurrent.TimeUnit.SECONDS); // Paper
- // Gale end - centralized async execution - remove Paper async executor
- } catch (java.lang.InterruptedException ignored) {} // Paper
if (org.spigotmc.SpigotConfig.saveUserCacheOnStopOnly) {
MinecraftServer.LOGGER.info("Saving usercache.json");
this.getProfileCache().save(false); // Paper
@@ -1008,6 +1003,13 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
LOGGER.info("Flushing Chunk IO");
io.papermc.paper.chunk.system.io.RegionFileIOThread.close(true); // Paper // Paper - rewrite chunk system
LOGGER.info("Closing Thread Pool");
+ // Gale start - centralized async execution - remove Paper async executor, remove background executor
+ AsyncExecutor.instance.shutdown(); // Paper
+ try {
+ AsyncExecutor.instance.awaitTermination(30, java.util.concurrent.TimeUnit.SECONDS); // Paper
+ // Gale end - centralized async execution - remove Paper async executor
+ } catch (java.lang.InterruptedException ignored) {} // Paper
+ // Gale start - centralized async execution - remove background executor
Util.shutdownExecutors(); // Paper
LOGGER.info("Closing Server");
try {
diff --git a/src/main/java/org/galemc/gale/concurrent/AsyncExecutor.java b/src/main/java/org/galemc/gale/concurrent/AsyncExecutor.java
index de182b9473963b95085fa612f70884a56765ae43..fff4549d86e672dc7b9959ac5dd51fd04d4d62c3 100644
--- a/src/main/java/org/galemc/gale/concurrent/AsyncExecutor.java
+++ b/src/main/java/org/galemc/gale/concurrent/AsyncExecutor.java
@@ -34,6 +34,7 @@ public final class AsyncExecutor extends ThreadPoolExecutor {
public static final int parallelism;
static {
int parallelismByEnvironmentVariable = Integer.getInteger("gale.threads.async", -1);
+ parallelismByEnvironmentVariable = Math.max(parallelismByEnvironmentVariable, Integer.getInteger("Paper.WorkerThreadCount", -1)); // Gale - centralized async execution - remove background executor
parallelism = Math.max(1, parallelismByEnvironmentVariable > 0 ? parallelismByEnvironmentVariable : CPUCoresEstimation.get() - 2);
}

29
patches/server/0027-Remove-bootstrap-executor.patch Normal file
View File

@@ -0,0 +1,29 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: MartijnMuijsers <martijnmuijsers@live.nl>
Date: Tue, 29 Nov 2022 01:23:49 +0100
Subject: [PATCH] Remove bootstrap executor
License: GPL-3.0 (https://www.gnu.org/licenses/gpl-3.0.html)
This patch was taken from Gale.
This patch is based on the following patch:
"completely remove bootstrapExecutor"
By: foss-mc <69294560+foss-mc@users.noreply.github.com>
As part of: Patina (https://github.com/PatinaMC/Patina)
Licensed under: GPL-3.0 (https://www.gnu.org/licenses/gpl-3.0.html)
diff --git a/src/main/java/net/minecraft/Util.java b/src/main/java/net/minecraft/Util.java
index e14245a77b40fca4bacf82295ad390339aba08a9..d84c721ed64cf927ccc9884235b22414298ce3ed 100644
--- a/src/main/java/net/minecraft/Util.java
+++ b/src/main/java/net/minecraft/Util.java
@@ -80,8 +80,8 @@ public class Util {
private static final int DEFAULT_MAX_THREADS = 255;
private static final String MAX_THREADS_SYSTEM_PROPERTY = "max.bg.threads";
private static final AtomicInteger WORKER_COUNT = new AtomicInteger(1);
- private static final ExecutorService BOOTSTRAP_EXECUTOR = makeExecutor("Bootstrap", -2); // Paper - add -2 priority
private static final ExecutorService BACKGROUND_EXECUTOR = AsyncExecutor.instance; // Gale - centralized async execution - remove background executor
+ private static final ExecutorService BOOTSTRAP_EXECUTOR = BACKGROUND_EXECUTOR; // Gale - Patina - remove bootstrap executor
// Paper start - don't submit BLOCKING PROFILE LOOKUPS to the world gen thread
public static final ExecutorService PROFILE_EXECUTOR = Executors.newFixedThreadPool(2, new java.util.concurrent.ThreadFactory() {

41
patches/server/0028-Remove-world-upgrade-executors.patch Normal file
View File

@@ -0,0 +1,41 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: MartijnMuijsers <martijnmuijsers@live.nl>
Date: Tue, 29 Nov 2022 14:44:23 +0100
Subject: [PATCH] Remove world upgrade executors
License: AGPL-3.0 (https://www.gnu.org/licenses/agpl-3.0.html)
This patch was taken from Gale.
diff --git a/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java
index 95cac7edae8ac64811fc6a2f6b97dd4a0fceb0b0..29a33ec067b9f47a7a86cce90316e1837d7d8612 100644
--- a/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java
+++ b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java
@@ -18,6 +18,8 @@ import net.minecraft.world.level.storage.DimensionDataStorage;
import net.minecraft.world.level.storage.LevelStorageSource;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import org.galemc.gale.concurrent.AsyncExecutor;
+
import java.io.File;
import java.io.IOException;
import java.text.DecimalFormat;
@@ -46,6 +48,10 @@ public class ThreadedWorldUpgrader {
this.dimensionType = dimensionType;
this.worldName = worldName;
this.worldDir = worldDir;
+ // Gale start - centralized async execution - remove world upgrade executors
+ this.threadPool = AsyncExecutor.instance;
+ /*
+ // Gale end - centralized async execution - remove world upgrade executors
this.threadPool = Executors.newFixedThreadPool(Math.max(1, threads), new ThreadFactory() {
private final AtomicInteger threadCounter = new AtomicInteger();
@@ -61,6 +67,7 @@ public class ThreadedWorldUpgrader {
return ret;
}
});
+ */ // Gale - centralized async execution - remove world upgrade executors
this.dataFixer = dataFixer;
this.generatorKey = generatorKey;
this.removeCaches = removeCaches;

31
patches/server/0029-Remove-tab-complete-executor.patch Normal file
View File

@@ -0,0 +1,31 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: MartijnMuijsers <martijnmuijsers@live.nl>
Date: Tue, 29 Nov 2022 14:58:26 +0100
Subject: [PATCH] Remove tab complete executor
License: AGPL-3.0 (https://www.gnu.org/licenses/agpl-3.0.html)
This patch was taken from Gale.
diff --git a/src/main/java/net/minecraft/server/network/ServerGamePacketListenerImpl.java b/src/main/java/net/minecraft/server/network/ServerGamePacketListenerImpl.java
index 103616e0dd5f7a5517eb20529263ac3ca06dd786..5a5d3b810f259725329dd90112d74c7f2d093c47 100644
--- a/src/main/java/net/minecraft/server/network/ServerGamePacketListenerImpl.java
+++ b/src/main/java/net/minecraft/server/network/ServerGamePacketListenerImpl.java
@@ -186,6 +186,7 @@ import net.minecraft.world.phys.shapes.Shapes;
import net.minecraft.world.phys.shapes.VoxelShape;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
+import org.galemc.gale.concurrent.AsyncExecutor; // Gale
// CraftBukkit start
import io.papermc.paper.adventure.ChatProcessor; // Paper
@@ -907,8 +908,7 @@ public class ServerGamePacketListenerImpl implements ServerPlayerConnection, Tic
}
// Paper start
- private static final java.util.concurrent.ExecutorService TAB_COMPLETE_EXECUTOR = java.util.concurrent.Executors.newFixedThreadPool(4,
- new com.google.common.util.concurrent.ThreadFactoryBuilder().setDaemon(true).setNameFormat("Async Tab Complete Thread - #%d").setUncaughtExceptionHandler(new net.minecraft.DefaultUncaughtExceptionHandlerWithName(net.minecraft.server.MinecraftServer.LOGGER)).build());
+ private static final java.util.concurrent.ExecutorService TAB_COMPLETE_EXECUTOR = AsyncExecutor.instance; // Gale - centralized async execution - remove tab complete executor
// Paper end
@Override
public void handleCustomCommandSuggestions(ServerboundCommandSuggestionPacket packet) {

30
patches/server/0030-Remove-text-filter-executor.patch Normal file
View File

@@ -0,0 +1,30 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: MartijnMuijsers <martijnmuijsers@live.nl>
Date: Tue, 29 Nov 2022 15:15:10 +0100
Subject: [PATCH] Remove text filter executor
License: AGPL-3.0 (https://www.gnu.org/licenses/agpl-3.0.html)
This patch was taken from Gale.
diff --git a/src/main/java/net/minecraft/server/network/TextFilterClient.java b/src/main/java/net/minecraft/server/network/TextFilterClient.java
index 92a60fc35145b7dd5fe17ea9b6823a919f483945..5922f569dcb6ecdacc9862749b393386a82c69fb 100644
--- a/src/main/java/net/minecraft/server/network/TextFilterClient.java
+++ b/src/main/java/net/minecraft/server/network/TextFilterClient.java
@@ -32,6 +32,7 @@ import net.minecraft.Util;
import net.minecraft.network.chat.FilterMask;
import net.minecraft.util.GsonHelper;
import net.minecraft.util.thread.ProcessorMailbox;
+import org.galemc.gale.concurrent.AsyncExecutor;
import org.slf4j.Logger;
public class TextFilterClient implements AutoCloseable {
@@ -62,7 +63,7 @@ public class TextFilterClient implements AutoCloseable {
this.joinEncoder = joinEncoder;
this.leaveEndpoint = leaveEndpoint;
this.leaveEncoder = leaveEncoder;
- this.workerPool = Executors.newFixedThreadPool(parallelism, THREAD_FACTORY);
+ this.workerPool = AsyncExecutor.instance; // Gale - centralized async execution - remove text filter executor
}
private static URL getEndpoint(URI root, @Nullable JsonObject endpoints, String key, String fallback) throws MalformedURLException {