'mkindex' to eventually replace go code; check from CI that output is…

… the same (crowdsecurity#1208)

* 'mkindex' to eventually replace go code; check from CI that output is the same

* remove classification=null

.github/workflows/update-index.yml

@@ -30,7 +30,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-go@v1
+      - uses: actions/setup-go@v5
         with:
           go-version: 1.19
       - uses: actions/checkout@v4
@@ -54,28 +54,54 @@ jobs:
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           branch: ${{ github.ref }}
-  
+
   run-linter:
     runs-on: ubuntu-latest
     needs: build
     steps:
+      - name: checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.ref }}
       - name: set up python
         uses: actions/setup-python@v5
         with:
           python-version: '3.12'
+          cache: 'pipenv'
+      - name: install dependencies
+        run: |
+          python -m pip install --upgrade pipenv wheel
+          pipenv install --deploy
+      - name: hub linter
+        run: |
+          pipenv run ./hublint defaults > .hublint.toml
+          pipenv run ./hublint check --color always --no-warning-details
+
+  run-mkindex:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
       - name: checkout repo
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
           ref: ${{ github.ref }}
-      - name: install pipenv
-        run: |
-          python -m pip install --upgrade pip
-          pip install pipenv
+      - name: set up python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: 'pipenv'
       - name: install dependencies
         run: |
-          pipenv install --deploy --python 3.12
-      - name: hub linter
+          python -m pip install --upgrade pipenv wheel
+          pipenv install --deploy
+      - name: run mkindex
         run: |
-          pipenv run ./hublint defaults > .hublint.toml
-          pipenv run ./hublint check --color always --no-warning-details
+          jq --sort-keys < .index.json >index.json
+          pipenv run ./mkindex >index-new.json
+          if ! (cmp -s index.json index-new.json); then
+              echo "Error: output from mkindex differs from legacy index builder."
+              diff -u index.json index-new.json
+              exit 1
+          fi

.index.json

@@ -16219,9 +16219,6 @@
    "content": "IyBmb3IgbWF4ICgxKSBwcmlvcml0eSA6IGtpbGwgb24gc2lnaHQKdHlwZTogdHJpZ2dlcgpuYW1lOiBjcm93ZHNlY3VyaXR5L3N1cmljYXRhLW1ham9yLXNldmVyaXR5CmRlc2NyaXB0aW9uOiAiRGV0ZWN0IGV4cGxvaXQgYXR0ZW1wdHMgdmlhIGVtZXJnaW5nIHRocmVhdCBydWxlcyIKZmlsdGVyOiAiZXZ0Lk1ldGEubG9nX3R5cGUgPT0gJ3N1cmljYXRhX2FsZXJ0JyAmJiBldnQuUGFyc2VkLnByb3RvID09ICdUQ1AnICYmIGV2dC5NZXRhLnN1cmljYXRhX3J1bGVfc2V2ZXJpdHkgPT0gJzEnIgpncm91cGJ5OiBldnQuTWV0YS5zb3VyY2VfaXAKYmxhY2tob2xlOiAxbQpyZXByb2Nlc3M6IHRydWUKbGFiZWxzOgogIHNlcnZpY2U6IHN1cmljYXRhCiAgcmVtZWRpYXRpb246IHRydWUKICBjb25maWRlbmNlOiAxCiAgc3Bvb2ZhYmxlOiAzCiAgY2xhc3NpZmljYXRpb246CiAgICAtIGF0dGFjay5UMTE5MAogICAgLSBhdHRhY2suVDE1OTUKICBiZWhhdmlvcjogImdlbmVyaWM6ZXhwbG9pdCIKICBsYWJlbDogIlN1cmljYXRhIFNldmVyaXR5IDEgRXZlbnQiCgotLS0KIyBmb3IgbG93ZXIgKDIpIHByaW9yaXR5IDogd2FpdCBmb3IgPj0zIGRpZmZlcmVudCBzaWduYXR1cmVzIGJlaW5nIHRyaWdnZXJlZAojIHdlIGludGVudGlvbmFsbHkgYXZvaWQgc2NlbmFyaW9zIG9uIHByaW9yaXR5IDMgYW5kIHN1Y2ggdGhhdCBhcmUgdG9vIHNlbnNpdGl2ZSB0byBmYWxzZSBwb3NpdGl2ZXMKdHlwZTogbGVha3kKY2FwYWNpdHk6IDIKbGVha3NwZWVkOiAyMHMKZGlzdGluY3Q6IGV2dC5NZXRhLnN1cmljYXRhX2FsZXJ0X3NpZ25hdHVyZV9pZApuYW1lOiBjcm93ZHNlY3VyaXR5L3N1cmljYXRhLWhpZ2gtbWVkaXVtLXNldmVyaXR5CmRlc2NyaXB0aW9uOiAiRGV0ZWN0IGV4cGxvaXQgYXR0ZW1wdHMgdmlhIGVtZXJnaW5nIHRocmVhdCBydWxlcyIKZmlsdGVyOiAiZXZ0Lk1ldGEubG9nX3R5cGUgPT0gJ3N1cmljYXRhX2FsZXJ0JyAmJiBldnQuUGFyc2VkLnByb3RvID09ICdUQ1AnICYmIGV2dC5NZXRhLnN1cmljYXRhX3J1bGVfc2V2ZXJpdHkgPT0gJzInIgpncm91cGJ5OiBldnQuTWV0YS5zb3VyY2VfaXAKYmxhY2tob2xlOiAxbQpyZXByb2Nlc3M6IHRydWUKbGFiZWxzOgogIHNlcnZpY2U6IHN1cmljYXRhCiAgY29uZmlkZW5jZTogMQogIHNwb29mYWJsZTogMwogIGNsYXNzaWZpY2F0aW9uOgogICAgLSBhdHRhY2suVDExOTAKICAgIC0gYXR0YWNrLlQxNTk1CiAgYmVoYXZpb3I6ICJnZW5lcmljOmV4cGxvaXQiCiAgbGFiZWw6ICJTdXJpY2F0YSBTZXZlcml0eSAyIEV2ZW50Igo=",
    "description": "Detect exploit attempts via emerging threat rules",
    "author": "crowdsecurity",
-   "references": [
-    "http://rules.emergingthreats.net/"
-   ],
    "labels": {
     "behavior": "generic:exploit",
     "classification": [

Pipfile

@@ -1,9 +1,11 @@
 [packages]
 pyyaml = "6.0.1"
 jsonschema = "4.21.1"
-requests = "2.31.0"
+requests = "2.32.3"
+PyGithub = "2.4.0"
 
 [dev-packages]
+black = "24.10.0"
 
 [requires]
-python_version = "*"
+python_version = "3.12"