Add build-with-entropy pipeline (#1384)

* Try entropy build

* Fix

* Updates

* Remove file

* Fix

* Add seed parameter to entropy.sh

* echo SEED used

* Try build pipeline changes

* Add python setup

* Fix pipeline

* Increase number of samples, add entropy to isle and config

* Try 32 samples

* Empty commit for another CI run

* Try 50 samples

* Empty commit for another CI run

* Empty commit for another CI run

* Empty commit for another CI run

* Empty commit for another CI run

* Empty commit for another CI run

* Empty commit for another CI run

* Empty commit for another CI run

* Empty commit for another CI run

* Trigger CI with 100 entropy samples

* Trigger CI with 100 entropy samples

* Trigger CI with 100 entropy samples

* Trigger CI with 100 entropy samples

* Trigger CI with 100 entropy samples

* Trigger CI with 100 entropy samples

* Trigger CI with 100 entropy samples

* Trigger CI with 100 entropy samples

* Trigger CI with 100 entropy samples

* Trigger CI with 100 entropy samples

* Update

* Echo seed

* Update text

* Fix requirements.txt

* Cancel previous CI workflows

* Try only build,yml

* Empty commit - should stop earlier build

* Fix upload

.github/workflows/build.yml

@@ -2,6 +2,10 @@ name: Build
 
 on: [push, pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   fetch-deps:
     name: Download original binaries
@@ -100,9 +104,104 @@ jobs:
           build/LEGO1.DLL
           build/LEGO1.PDB
 
+  build-with-entropy:
+    name: 'MSVC 4.20 with entropy'
+    needs: [fetch-deps]
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        instance: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
+                   10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
+                   20, 21, 22, 23, 24, 25, 26, 27, 28, 29]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - uses: actions/checkout@v4
+      with:
+        repository: itsmattkc/msvc420
+        path: msvc420
+
+    - name: Setup cmake
+      uses: jwlawson/actions-setup-cmake@v2
+      with:
+        # Use minimum supported version
+        cmake-version: '3.15.x'
+
+    - name: Patch MSVC 4.2
+      run: |
+        tools/patch_c2.py msvc420/bin/C2.EXE
+
+    - name: Generate Entropy
+      shell: bash
+      run: |
+        # Get the first 8 characters of the SHA (enough for a decent seed)
+        SHA_PREFIX=$(echo "${{ github.sha }}" | cut -c 1-8)
+        ENTROPY_SEED=$((16#$SHA_PREFIX + ${{ matrix.instance }}))
+
+        echo "Using seed: $ENTROPY_SEED"
+        python3 tools/entropy.py $ENTROPY_SEED > entropy.h
+
+    - name: Build
+      shell: cmd
+      run: |
+        call .\msvc420\bin\VCVARS32.BAT x86
+        cmake -B build -DCMAKE_BUILD_TYPE=RelWithDebInfo -DISLE_INCLUDE_ENTROPY=ON -G "NMake Makefiles"
+        cmake --build build
+
+    - name: Restore cached original binaries
+      id: cache-original-binaries
+      uses: actions/cache/restore@v4
+      with:
+        enableCrossOsArchive: true
+        path: legobin
+        key: legobin
+        
+    - name: Install python packages
+      shell: bash
+      run: |
+        pip install -r tools/requirements.txt
+  
+    - name: Detect binaries
+      run: |
+        reccmp-project detect --what original   --search-path legobin
+        reccmp-project detect --what recompiled --search-path build
+
+    - name: Summarize Accuracy
+      shell: bash
+      run: |
+        reccmp-reccmp --target CONFIG --json CONFIGPROGRESS.json
+        reccmp-reccmp --target ISLE --json ISLEPROGRESS.json
+        reccmp-reccmp --target LEGO1 --json LEGO1PROGRESS.json
+
+    - name: Upload Artifact
+      uses: actions/upload-artifact@main
+      with:
+        name: Win32-Entropy-${{ matrix.instance }}
+        path: |
+          CONFIGPROGRESS.json
+          ISLEPROGRESS.json
+          LEGO1PROGRESS.json
+
+  merge-entropy-artifacts:
+    name: 'Merge entropy artifacts'
+    runs-on: ubuntu-latest
+    needs: build-with-entropy
+    steps:
+      - name: Merge Artifacts
+        uses: actions/upload-artifact/merge@v4
+        with:
+          name: Win32-Entropy
+          pattern: Win32-Entropy-*
+          separate-directories: true
+          
   compare:
     name: Compare with master
-    needs: [build, fetch-deps]
+    needs: [build, merge-entropy-artifacts, fetch-deps]
     runs-on: windows-latest
     steps:
     - uses: actions/checkout@main
@@ -113,8 +212,13 @@ jobs:
 
     - uses: actions/download-artifact@main
       with:
-       name: Win32
+        name: Win32
-       path: build
+        path: build
+
+    - uses: actions/download-artifact@main
+      with:
+        name: Win32-Entropy
+        path: build-entropy
 
     - name: Restore cached original binaries
       id: cache-original-binaries
@@ -156,6 +260,28 @@ jobs:
         reccmp-reccmp --target ISLE --diff ISLEPROGRESS-old.json || echo "Current master not found"
         reccmp-reccmp --target LEGO1 --diff LEGO1PROGRESS-old.json || echo "Current master not found"
 
+    - name: Aggregate Accuracy
+      shell: bash
+      run: |
+        reccmp-aggregate --samples $(find build-entropy -type f -name "CONFIGPROGRESS.json") --output CONFIGPROGRESS-agg.json
+        reccmp-aggregate --samples $(find build-entropy -type f -name "ISLEPROGRESS.json") --output ISLEPROGRESS-agg.json
+        reccmp-aggregate --samples $(find build-entropy -type f -name "LEGO1PROGRESS.json") --output LEGO1PROGRESS-agg.json
+
+    - name: Compare Aggregate Accuracy With Current Master
+      shell: bash
+      env:
+        RELEASE_URL: https://github.com/isledecomp/isle/releases/download/continuous
+      run: |
+        # Download the current master state
+        curl -fLSs -o CONFIGPROGRESS-agg-old.json $RELEASE_URL/CONFIGPROGRESS-agg.json || echo "" >CONFIGPROGRESS-agg-old.json
+        curl -fLSs -o ISLEPROGRESS-agg-old.json $RELEASE_URL/ISLEPROGRESS-agg.json || echo "" >ISLEPROGRESS-agg-old.json
+        curl -fLSs -o LEGO1PROGRESS-agg-old.json $RELEASE_URL/LEGO1PROGRESS-agg.json || echo "" >LEGO1PROGRESS-agg-old.json
+        
+        # Compare with current master
+        reccmp-aggregate --diff CONFIGPROGRESS-agg-old.json CONFIGPROGRESS-agg.json || echo "Current master not found"
+        reccmp-aggregate --diff ISLEPROGRESS-agg-old.json ISLEPROGRESS-agg.json || echo "Current master not found"
+        reccmp-aggregate --diff LEGO1PROGRESS-agg-old.json LEGO1PROGRESS-agg.json || echo "Current master not found"
+
     - name: Test Exports
       shell: bash
       run: |
@@ -180,9 +306,9 @@ jobs:
       with:
         name: Accuracy Report
         path: |
-          CONFIGPROGRESS.*
+          CONFIGPROGRESS*
-          ISLEPROGRESS.*
+          ISLEPROGRESS*
-          LEGO1PROGRESS.*
+          LEGO1PROGRESS*
 
   upload:
     name: Upload artifacts
@@ -212,9 +338,9 @@ jobs:
           build/CONFIG.EXE \
           build/ISLE.EXE \
           build/LEGO1.DLL \
-          CONFIGPROGRESS.* \
+          CONFIGPROGRESS* \
-          ISLEPROGRESS.* \
+          ISLEPROGRESS* \
-          LEGO1PROGRESS.*
+          LEGO1PROGRESS*
 
         curl -X POST -F key=$UPLOAD_KEY -F 'file=@CONFIGPROGRESS.SVG' https://legoisland.org/progress/
         curl -X POST -F key=$UPLOAD_KEY -F 'file=@ISLEPROGRESS.SVG' https://legoisland.org/progress/

CMakeLists.txt

@@ -66,6 +66,7 @@ option(ISLE_DECOMP_ASSERT "Assert struct size" ${MSVC_FOR_DECOMP})
 cmake_dependent_option(ISLE_USE_DX5_LIBS "Build with internal DirectX 5 SDK Libraries" ON ISLE_USE_DX5 OFF)
 option(ISLE_BUILD_LEGO1 "Build LEGO1.DLL library" ON)
 option(ISLE_BUILD_BETA10 "Build BETA10.DLL library" OFF)
+option(ISLE_INCLUDE_ENTROPY "Build with entropy.h" OFF)
 
 if(NOT (ISLE_BUILD_LEGO1 OR ISLE_BUILD_BETA10))
   message(FATAL_ERROR "ISLE_BUILD_LEGO1 AND ISLE_BUILD_BETA10 cannot be both disabled")
@@ -584,6 +585,18 @@ if (MSVC_FOR_DECOMP)
     set_property(TARGET isle ${lego1_targets} PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
   endif()
 
+  if (ISLE_INCLUDE_ENTROPY)
+    foreach(tgt IN LISTS lego1_targets beta10_targets)
+      target_compile_options(${tgt} PRIVATE /FI${PROJECT_SOURCE_DIR}/entropy.h)
+    endforeach()
+    if (TARGET isle)
+      target_compile_options(isle PRIVATE /FI${PROJECT_SOURCE_DIR}/entropy.h)
+    endif()
+    if (TARGET config)
+      target_compile_options(config PRIVATE /FI${PROJECT_SOURCE_DIR}/entropy.h)
+    endif()
+  endif()
+
   if(TARGET lego1)
     target_link_options(lego1 PRIVATE "/OPT:REF")
     # Equivalent to target_compile_options(... PRIVATE "/MT$<$<CONFIG:Debug>:d>")

tools/entropy.py

@@ -0,0 +1,45 @@
+import random
+import string
+import sys
+
+# Parameters for tweaking:
+MAX_CLASSES = 10
+MAX_FUNC_PER_CLASS = 10
+
+# Only the unique suffix, not counting "Class" or "Function"
+CLASS_NAME_LEN = 6
+FUNC_NAME_LEN = 8
+
+
+def random_camel_case(length: int) -> str:
+    """Return a random string with first letter capitalized."""
+    return "".join(
+        [
+            random.choice(string.ascii_uppercase),
+            *random.choices(string.ascii_lowercase, k=length - 1),
+        ]
+    )
+
+
+# If the first parameter is an integer, use it as the seed.
+try:
+    seed = int(sys.argv[1])
+except (IndexError, ValueError):
+    seed = random.randint(0, 10000)
+
+random.seed(seed)
+
+print(f"// Seed: {seed}\n")
+
+num_classes = random.randint(1, MAX_CLASSES)
+for i in range(num_classes):
+    class_name = "Class" + random_camel_case(CLASS_NAME_LEN)
+    print(f"class {class_name} {{")
+    num_functions = random.randint(1, MAX_FUNC_PER_CLASS)
+    for j in range(num_functions):
+        function_name = "Function" + random_camel_case(FUNC_NAME_LEN)
+        print(f"\tinline void {function_name}() {{}}")
+
+    print(f"}};\n")
+
+print()