Added flush threshold for linear region file

.github/workflows/build_1.20.4.yml

@@ -1,71 +1,66 @@
-name: Luminol CI v2 - Ver/1.20.4
-
-on:
-    push:
-      branches: [ "ver/1.20.4" ]
-    pull_request:
-      branches: [ "ver/1.20.4" ]
-
-permissions: write-all
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    environment: default
-    steps:
-      - name: Checkout Git Repository
-        uses: actions/checkout@v4
-      - name: Validate Gradle wrapper
-        uses: gradle/actions/wrapper-validation@v3
-      - name: Setup Gradle
-        uses: gradle/actions/setup-gradle@v3
-      - name: Set up JDK
-        uses: actions/setup-java@v4
-        with:
-          distribution: 'temurin'
-          java-version: '17'
-      - name: Configure Git User Details
-        run: git config --global user.email "ci@luminolmc.com" && git config --global user.name "LuminolMC CI"
-      - name: Apply Patches
-        uses: gradle/actions/setup-gradle@v3
-        with:
-          arguments: applyPatches
-      - name: CreateJar
-        uses: gradle/actions/setup-gradle@v3
-        with:
-          arguments: createReobfBundlerJar createReobfPaperclipJar
-      - name: SetENV
-        run: |
-          sh scripts/SetENV.sh
-          echo "${{ env.tag }}"
-      - name: Upload Artifact
-        uses: "actions/upload-artifact@v4"
-        with:
-          name: "${{ env.project_id_b }} CI Artifacts"
-          path: "build/libs/*.jar"
-      - name: Create Release
-        if: "!contains(github.event.commits[0].message, '[release skip]')"
-        uses: ncipollo/release-action@v1.14.0
-        with:
-          tag: ${{ env.tag }}
-          name: ${{ env.project_id_b }} ${{ env.mcversion }} - ${{ env.commit_id }}
-          body: |
-            ====
-            📦Version: `${{ env.mcversion }}` | Commit ${{ env.commit_id }} [![download](https://img.shields.io/github/downloads/LuminolMC/${{ env.project_id }}/${{ env.tag }}/total?color=0)](https://github.com/LuminolMC/${{ env.project_id }}/download/${{ env.tag }}/${{ env.jar }})
-            This release is automatically compiled by GitHub Actions
-            ### Commit Message
-            ${{ env.commit_msg }}
-          artifacts: |
-            ${{ env.jar_dir }}
-            build/libs/${{ env.project_id }}-${{ env.mcversion }}-bundler.jar
-          generateReleaseNotes: true
-          prerelease: ${{ env.pre }}
-          makeLatest: ${{ env.make_latest }}
-          token: "${{ secrets.GITHUB_TOKEN }}"
-      - name: Push to API
-        if: "!contains(github.event.commits[0].message, '[release skip]')"
-        env:
-          secret_v2: ${{ secrets.API_KEY }}
-          tag: ${{ env.tag }}
-          project_id: ${{ env.project_id }}
+name: Luminol CI - ver/1.20.4
+
+on:
+    push:
+      branches: [ "ver/1.20.4" ]
+    pull_request:
+      branches: [ "ver/1.20.4" ]
+
+permissions: write-all
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    environment: default
+    steps:
+      - name: Checkout Git Repository
+        uses: actions/checkout@v4
+      - name: Validate Gradle wrapper
+        uses: gradle/actions/wrapper-validation@v3
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v3
+        with:
+          gradle-version: 8.4
+      - name: Set up JDK
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+      - name: Configure Git User Details
+        run: git config --global user.email "ci@luminolmc.com" && git config --global user.name "LuminolMC CI"
+      - name: Apply Patches
+        run: gradle applyPatches
+      - name: CreateJar
+        run: gradle createReobfBundlerJar createReobfPaperclipJar
+      - name: SetENV
+        run: sh scripts/SetENV.sh
+      - name: Upload Artifact
+        uses: "actions/upload-artifact@v4"
+        with:
+          name: "${{ env.project_id_b }} CI Artifacts"
+          path: "build/libs/*.jar"
+      - name: Create Release
+        if: "!contains(github.event.commits[0].message, '[release skip]')"
+        uses: ncipollo/release-action@v1.14.0
+        with:
+          tag: ${{ env.tag }}
+          name: ${{ env.project_id_b }} ${{ env.mcversion }} - ${{ env.commit_id }}
+          body: |
+            📦Version: `${{ env.mcversion }}` | Commit ${{ env.commit_id }} [![download](https://img.shields.io/github/downloads/LuminolMC/${{ env.project_id }}/${{ env.tag }}/total?color=0&style=flat-square)](https://github.com/LuminolMC/${{ env.project_id }}/download/${{ env.tag }}/${{ env.jar }})
+            This release is automatically compiled by GitHub Actions
+            ### Commit Message
+            ${{ env.commit_msg }}
+          artifacts: |
+            ${{ env.jar_dir }}
+            build/libs/${{ env.project_id }}-${{ env.mcversion }}-bundler.jar
+          generateReleaseNotes: true
+          prerelease: ${{ env.pre }}
+          makeLatest: ${{ env.make_latest }}
+          token: "${{ secrets.GITHUB_TOKEN }}"
+      - name: Push to API
+        if: "!contains(github.event.commits[0].message, '[release skip]')"
+        env:
+          secret_v2: ${{ secrets.API_KEY }}
+          tag: ${{ env.tag }}
+          project_id: ${{ env.project_id }}
         run: sh scripts/PushToAPI.sh

.github/workflows/build_1_20_4.yml_dis

@@ -1,63 +0,0 @@
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
-# This workflow will build a Java project with Gradle and cache/restore any dependencies to improve the workflow execution time
-# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-gradle
-
-name: LuminolCI - Ver/1.20.4
-
-on:
-  push:
-    branches: [ "ver/1.20.4" ]
-  pull_request:
-    branches: [ "ver/1.20.4" ]
-
-permissions: write-all
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up git
-      run: git config --global user.email "noreply@github.com" && git config --global user.name "ci"
-    - name: Set up JDK 17
-      uses: actions/setup-java@v3
-      with:
-        java-version: '17'
-        distribution: 'temurin'
-    - name: Make gradlew executable
-      run: chmod 777 ./gradlew
-    - name: Setup project
-      uses: gradle/gradle-build-action@bd5760595778326ba7f1441bcf7e88b49de61a25 # v2.6.0
-      with:
-        arguments: applyPatches
-    - name: Build project to paperclip jar
-      uses: gradle/gradle-build-action@bd5760595778326ba7f1441bcf7e88b49de61a25 # v2.6.0
-      with:
-        arguments: createReobfBundlerJar createReobfPaperclipJar
-    - name: Capture build artifacts
-      uses: actions/upload-artifact@v3
-      with:
-        name: Artifacts
-        path: build/libs/
-    - name: Publish All
-      run: |
-        echo "GITHUB_USERNAME=LuminolCI" >> $GITHUB_ENV
-        export GITHUB_USERNAME=LuminolCI
-        echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> $GITHUB_ENV
-        export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
-        ./gradlew publish -PpublishDevBundle=true
-    - name: Rename jar file
-      run: mv build/libs/luminol-paperclip-1.20.4-R0.1-SNAPSHOT-reobf.jar build/libs/luminol-1.20.4-paperclip.jar
-    - name: Release Artifacts
-      uses: svenstaro/upload-release-action@v2
-      with:
-       release_name: "Luminol MC1.20.4 - ${{ github.event.repository.updated_at}}"
-       tag: "1.20.4-${{ github.run_id }}"
-       repo_token: "${{ secrets.GITHUB_TOKEN }}"
-       file: "build/libs/luminol-1.20.4-paperclip.jar"
-       file_glob: true
-       prerelease: true

gradle.properties

@@ -8,6 +8,6 @@ org.gradle.parallel = true
 org.gradle.vfs.watch = false
 org.gradle.jvmargs = -Xmx3G
 
-mcVersion=1.20.4
-GroupMCV=1.20
-preVersion=false
+mcVersion = 1.20.4
+GroupMCV = 1.20
+preVersion = false

patches/server/0017-Kaiiju-linear-region-format-and-settings.patch

@@ -93,10 +93,10 @@ index 0000000000000000000000000000000000000000..dcfbabf54b19a4c29d5c95830242c5c2
 +}
 diff --git a/src/main/java/dev/kaiijumc/kaiiju/region/LinearRegionFile.java b/src/main/java/dev/kaiijumc/kaiiju/region/LinearRegionFile.java
 new file mode 100644
-index 0000000000000000000000000000000000000000..dc5a5f42c9ca7a50295c18424722568a3ad33fa5
+index 0000000000000000000000000000000000000000..7e551a5fb7db6a0aa4f60b15587be2b716c54000
 --- /dev/null
 +++ b/src/main/java/dev/kaiijumc/kaiiju/region/LinearRegionFile.java
-@@ -0,0 +1,324 @@
+@@ -0,0 +1,341 @@
 +package dev.kaiijumc.kaiiju.region;
 +
 +import com.github.luben.zstd.ZstdInputStream;
@@ -120,6 +120,7 @@ index 0000000000000000000000000000000000000000..dc5a5f42c9ca7a50295c18424722568a
 +import java.util.Arrays;
 +import java.util.List;
 +import java.util.concurrent.atomic.AtomicBoolean;
++import java.util.concurrent.atomic.AtomicInteger;
 +import java.util.concurrent.locks.ReentrantLock;
 +
 +public class LinearRegionFile implements AbstractRegionFile, AutoCloseable {
@@ -129,7 +130,7 @@ index 0000000000000000000000000000000000000000..dc5a5f42c9ca7a50295c18424722568a
 +    private static final int FOOTER_SIZE = 8;
 +    private static final Logger LOGGER = LogUtils.getLogger();
 +    private static final List<Byte> SUPPORTED_VERSIONS = Arrays.asList((byte) 1, (byte) 2);
-+    private static final LinearRegionFileFlusher linearRegionFileFlusher = new LinearRegionFileFlusher();
++    public static final LinearRegionFileFlusher linearRegionFileFlusher = new LinearRegionFileFlusher();
 +
 +    private final byte[][] buffer = new byte[1024][];
 +    private final int[] bufferUncompressedSize = new int[1024];
@@ -147,6 +148,9 @@ index 0000000000000000000000000000000000000000..dc5a5f42c9ca7a50295c18424722568a
 +    public boolean closed = false;
 +    public Path path;
 +
++    public volatile boolean savingScheduled = true;
++    public final AtomicInteger dirtyChunkCount = new AtomicInteger(0);
++
 +
 +    public LinearRegionFile(Path file, int compression) throws IOException {
 +        this.path = file;
@@ -225,14 +229,20 @@ index 0000000000000000000000000000000000000000..dc5a5f42c9ca7a50295c18424722568a
 +    }
 +
 +    public void flush() throws IOException {
-+        if (getAndResetSaveMarker()) flushWrapper(); // sync
++        if (getAndResetSaveMarker()) {
++            this.savingScheduled = false; //cancel async saving
++            flushWrapper(); // sync
++        }
 +    }
 +
 +    private void markToSave() {
-+        linearRegionFileFlusher.scheduleSave(this);
 +        markedToSave.set(true);
 +    }
 +
++    private void signalToSave(){
++        linearRegionFileFlusher.scheduleSave(this);
++    }
++
 +    public boolean getAndResetSaveMarker() {
 +        return markedToSave.getAndSet(false);
 +    }
@@ -250,6 +260,8 @@ index 0000000000000000000000000000000000000000..dc5a5f42c9ca7a50295c18424722568a
 +            save();
 +        } catch (IOException e) {
 +            LOGGER.error("Failed to flush region file " + path.toAbsolutePath(), e);
++        }finally {
++            this.dirtyChunkCount.set(0); //Reset dirty chunk count
 +        }
 +    }
 +
@@ -333,8 +345,11 @@ index 0000000000000000000000000000000000000000..dc5a5f42c9ca7a50295c18424722568a
 +            this.bufferUncompressedSize[getChunkIndex(pos.x, pos.z)] = uncompressedSize;
 +        } catch (IOException e) {
 +            LOGGER.error("Chunk write IOException " + e + " " + this.path);
++        }finally {
++            this.dirtyChunkCount.getAndIncrement();
 +        }
 +        markToSave();
++        signalToSave();
 +    }
 +
 +    public DataOutputStream getChunkDataOutputStream(ChunkPos pos) {
@@ -386,7 +401,9 @@ index 0000000000000000000000000000000000000000..dc5a5f42c9ca7a50295c18424722568a
 +        this.buffer[i] = null;
 +        this.bufferUncompressedSize[i] = 0;
 +        this.chunkTimestamps[i] = getTimestamp();
++        this.dirtyChunkCount.getAndIncrement();
 +        markToSave();
++        signalToSave();
 +    }
 +
 +    public boolean hasChunk(ChunkPos pos) {
@@ -423,22 +440,19 @@ index 0000000000000000000000000000000000000000..dc5a5f42c9ca7a50295c18424722568a
 +}
 diff --git a/src/main/java/dev/kaiijumc/kaiiju/region/LinearRegionFileFlusher.java b/src/main/java/dev/kaiijumc/kaiiju/region/LinearRegionFileFlusher.java
 new file mode 100644
-index 0000000000000000000000000000000000000000..65435eb666b2e1cabffc145e016faa5e3b373464
+index 0000000000000000000000000000000000000000..dea4dffd67ee9881a224acf41367c6845acec832
 --- /dev/null
 +++ b/src/main/java/dev/kaiijumc/kaiiju/region/LinearRegionFileFlusher.java
-@@ -0,0 +1,57 @@
+@@ -0,0 +1,63 @@
 +package dev.kaiijumc.kaiiju.region;
 +
 +import com.google.common.util.concurrent.ThreadFactoryBuilder;
 +
-+import java.util.Set;
 +import java.util.concurrent.*;
 +import org.bukkit.Bukkit;
 +
 +public class LinearRegionFileFlusher {
-+    private final Set<LinearRegionFile> pendingSaving = ConcurrentHashMap.newKeySet();
-+
-+    private final ExecutorService executor = Executors.newFixedThreadPool(
++    private final ExecutorService ioExecutor = Executors.newFixedThreadPool(
 +        me.earthme.luminol.config.modules.misc.RegionFormatConfig.linearFlushThreads,
 +        new ThreadFactoryBuilder()
 +            .setNameFormat("linear-flusher-%d")
@@ -448,7 +462,7 @@ index 0000000000000000000000000000000000000000..65435eb666b2e1cabffc145e016faa5e
 +    private final Executor delayedFlusher = CompletableFuture.delayedExecutor(
 +            me.earthme.luminol.config.modules.misc.RegionFormatConfig.linearFlushFrequency,
 +            TimeUnit.SECONDS,
-+            executor
++            ioExecutor
 +    );
 +
 +    public LinearRegionFileFlusher() {
@@ -456,28 +470,37 @@ index 0000000000000000000000000000000000000000..65435eb666b2e1cabffc145e016faa5e
 +    }
 +
 +    public void scheduleSave(LinearRegionFile regionFile) {
-+        if (this.pendingSaving.contains(regionFile) || !regionFile.isMarkedToSave()){
++        if (regionFile.dirtyChunkCount.get() > me.earthme.luminol.config.modules.misc.RegionFormatConfig.flushImmediatelyIfDirtyChunkHigherThan){
++            this.scheduleSaveInternal(regionFile,this.ioExecutor);
 +            return;
 +        }
 +
-+        this.pendingSaving.add(regionFile);
-+        this.delayedFlusher.execute(()->{
++        this.scheduleSaveInternal(regionFile,this.delayedFlusher);
++    }
++
++    private void scheduleSaveInternal(LinearRegionFile regionFile,Executor scheduler){
++        if (!regionFile.savingScheduled || !regionFile.isMarkedToSave()){
++            return;
++        }
++
++        regionFile.savingScheduled = true;
++        scheduler.execute(()->{
 +            try {
 +                if (!regionFile.closed && regionFile.isMarkedToSave()){
 +                    regionFile.flushWrapper();
 +                }
 +            }finally {
 +                regionFile.resetSaveMarker();
-+                this.pendingSaving.remove(regionFile);
++                regionFile.savingScheduled = false;
 +            }
 +        });
 +    }
 +
 +    public void shutdown() {
-+        this.executor.shutdown();
++        this.ioExecutor.shutdown();
 +        for (;;) {
 +            try {
-+                if (this.executor.awaitTermination(5_00,TimeUnit.MILLISECONDS)) break;
++                if (this.ioExecutor.awaitTermination(5_00,TimeUnit.MILLISECONDS)) break;
 +            } catch (InterruptedException e) {
 +                e.printStackTrace();
 +            }
@@ -573,6 +596,18 @@ index 2934f0cf0ef09c84739312b00186c2ef0019a165..b46acbc078f3d3bfb0f3ede3f1cc172f
                  }
              }
          }
+diff --git a/src/main/java/io/papermc/paper/threadedregions/RegionShutdownThread.java b/src/main/java/io/papermc/paper/threadedregions/RegionShutdownThread.java
+index 3f3fd8b1881106f893ffb677272e50d77cbb6626..338dfc1c87ac82df80b3b87148deb791110ce3cc 100644
+--- a/src/main/java/io/papermc/paper/threadedregions/RegionShutdownThread.java
++++ b/src/main/java/io/papermc/paper/threadedregions/RegionShutdownThread.java
+@@ -164,6 +164,7 @@ public final class RegionShutdownThread extends TickThread {
+ 
+             this.saveLevelData(world);
+         }
++        dev.kaiijumc.kaiiju.region.LinearRegionFile.linearRegionFileFlusher.shutdown(); //Luminol - Kaiiju linear format fixes
+         // moved from stop part 1
+         // we need this to be after saving level data, as that will complete any teleportations the player is in
+         LOGGER.info("Saving players");
 diff --git a/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java
 index 9017907c0ec67a37a506f09b7e4499cef7885279..8b9ffcaab5d71660291d1c0454d2abd969d8c6ae 100644
 --- a/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java
@@ -594,10 +629,10 @@ index 9017907c0ec67a37a506f09b7e4499cef7885279..8b9ffcaab5d71660291d1c0454d2abd9
  
 diff --git a/src/main/java/me/earthme/luminol/config/modules/misc/RegionFormatConfig.java b/src/main/java/me/earthme/luminol/config/modules/misc/RegionFormatConfig.java
 new file mode 100644
-index 0000000000000000000000000000000000000000..5d594b606178bd26180caad49de17c0e67e89747
+index 0000000000000000000000000000000000000000..ab1c3e0276bbdbb32fd4155f7c596f3172c7f351
 --- /dev/null
 +++ b/src/main/java/me/earthme/luminol/config/modules/misc/RegionFormatConfig.java
-@@ -0,0 +1,58 @@
+@@ -0,0 +1,60 @@
 +package me.earthme.luminol.config.modules.misc;
 +
 +import com.electronwill.nightconfig.core.file.CommentedFileConfig;
@@ -620,6 +655,8 @@ index 0000000000000000000000000000000000000000..5d594b606178bd26180caad49de17c0e
 +    @HotReloadUnsupported
 +    @ConfigInfo(baseName = "linear_flush_frequency")
 +    public static int linearFlushFrequency = 10;
++    @ConfigInfo(baseName = "flush_immediately_if_queued_higher_than")
++    public static int flushImmediatelyIfDirtyChunkHigherThan = 5;
 +
 +    @Override
 +    public EnumConfigCategory getCategory() {

scripts/PushToAPI.sh

@@ -2,7 +2,7 @@ sha256() {
   sha256sum $1 | awk '{print $1}'
 }
 prop() {
-  grep "${1}" gradle.properties | cut -d'=' -f2 | sed 's/\r//'
+  grep "^[[:space:]]*${1}" gradle.properties | cut -d'=' -f2 | sed 's/^[[:space:]]*//; s/\r//'
 }
 
 mcversion_group=$(prop GroupMCV)
@@ -22,9 +22,6 @@ jar_name="$pro_id_mcv-paperclip.jar"
 ctime=$(date -u +"%s")"000"
 
 # v2
-# v2
-echo "Authentication: $secret_v2"
-#echo "{\"version_group\":\"$mcversion_group\",\"channel\":\"$channel\",\"changes\":\"$changes\",\"jar_name\":\"$jar_name\",\"sha256\":\"$jar_sha256\",\"release_tag\":\"$tag\",\"time\":\"$ctime\"}" > data.json
 json=$(printf "{\\\"channel\\\":\\\"%s\\\", \\\"jar_name\\\":\\\"%s\\\", \\\"sha256\\\":\\\"%s\\\", \\\"tag\\\":\\\"%s\\\", \\\"time\\\":\\\"%s\\\", \\\"changes\\\":\\\"%s\\\"}" \
     "$channel" "$jar_name" "$jar_sha256" "$tag" "$ctime" "$changes")
 echo "[DEBUG] $json"

scripts/SetENV.sh

@@ -1,5 +1,5 @@
 prop() {
-  grep "${1}" gradle.properties | cut -d'=' -f2 | sed 's/\r//'
+  grep "^[[:space:]]*${1}" gradle.properties | cut -d'=' -f2 | sed 's/^[[:space:]]*//; s/\r//'
 }
 
 project_id="luminol"
@@ -20,8 +20,7 @@ mv build/libs/$project_id-bundler-$grdversion-reobf.jar build/libs/$project_id-$
 echo "project_id=$project_id" >> $GITHUB_ENV
 echo "project_id_b=$project_id_b" >> $GITHUB_ENV
 echo "commit_id=$commitid" >> $GITHUB_ENV
-echo "commit_msg=$(git log --pretty='> [%h] %s' -$number)" >> $GITHUB_ENV
-echo "$commit_msg"
+echo "commit_msg=$(git log --pretty='> [%h] %s' -1)" >> $GITHUB_ENV
 echo "mcversion=$mcversion" >> $GITHUB_ENV
 echo "pre=$preVersion" >> $GITHUB_ENV
 echo "tag=$release_tag" >> $GITHUB_ENV