support for type "contexts" (#889)

* support for type "contexts" * Update blockers meta * Update index * add generic http context * Update index * fix context files * Update index * up * Update index --------- Co-authored-by: GitHub Action <[email protected]> Co-authored-by: bui <[email protected]>
crowdsecurity · Jan 18, 2024 · 58ac955 · 58ac955
1 parent b943c28
commit 58ac955
Show file tree

Hide file tree

Showing 11 changed files with 129 additions and 34 deletions.
diff --git a/.github/workflows/test_configurations.yaml b/.github/workflows/test_configurations.yaml
@@ -3,29 +3,33 @@ on:
   pull_request:
     branches: [ master ]
     paths:
-      - 'scenarios/**.yaml'
-      - 'parsers/**.yaml'
-      - 'postoverflows/**.yaml'
       - 'collections/**.yaml'
-      - 'scenarios/**.yml'
+      - 'collections/**.yml'
+      - 'contexts/**.yaml'
+      - 'contexts/**.yml'
+      - 'parsers/**.yaml'
       - 'parsers/**.yml'
+      - 'postoverflows/**.yaml'
       - 'postoverflows/**.yml'
-      - 'collections/**.yml'
+      - 'scenarios/**.yaml'
+      - 'scenarios/**.yml'
       - '.github/workflows/**.yaml'
       - '.github/workflows/**.yml'
       - '.tests/**'
       - '!.github/workflows/update_taxonomy.yaml'
   push:
     branches: [ master ]
     paths:
-      - 'scenarios/**.yaml'
-      - 'parsers/**.yaml'
-      - 'postoverflows/**.yaml'
       - 'collections/**.yaml'
-      - 'scenarios/**.yml'
+      - 'collections/**.yml'
+      - 'contexts/**.yaml'
+      - 'contexts/**.yml'
+      - 'parsers/**.yaml'
       - 'parsers/**.yml'
+      - 'postoverflows/**.yaml'
       - 'postoverflows/**.yml'
-      - 'collections/**.yml'
+      - 'scenarios/**.yaml'
+      - 'scenarios/**.yml'
       - '.github/workflows/**.yaml'
       - '.github/workflows/**.yml'
       - '.tests/**'

diff --git a/.github/workflows/update-index.yml b/.github/workflows/update-index.yml
@@ -3,24 +3,27 @@ name: Update index
 on:
   push:
     paths:
-      - 'scenarios/**.yaml'
-      - 'parsers/**.yaml'
-      - 'postoverflows/**.yaml'
-      - 'collections/**.yaml'
-      - 'appsec-rules/**.yaml'
+      - 'appsec-configs/**.md'
       - 'appsec-configs/**.yaml'
-      - 'scenarios/**.yml'
-      - 'parsers/**.yml'
-      - 'postoverflows/**.yml'
-      - 'collections/**.yml'
-      - 'appsec-rules/**.yml'
       - 'appsec-configs/**.yml'
-      - 'scenarios/**.md'
+      - 'appsec-rules/**.md'
+      - 'appsec-rules/**.yaml'
+      - 'appsec-rules/**.yml'
+      - 'collections/**.md'
+      - 'collections/**.yaml'
+      - 'collections/**.yml'
+      - 'contexts/**.md'
+      - 'contexts/**.yaml'
+      - 'contexts/**.yml'
       - 'parsers/**.md'
+      - 'parsers/**.yaml'
+      - 'parsers/**.yml'
       - 'postoverflows/**.md'
-      - 'collections/**.md'
-      - 'appsec-rules/**.md'
-      - 'appsec-configs/**.md'
+      - 'postoverflows/**.yaml'
+      - 'postoverflows/**.yml'
+      - 'scenarios/**.md'
+      - 'scenarios/**.yaml'
+      - 'scenarios/**.yml'
       - '.github/workflows/update-index.yml'
       - "*.go"
 jobs:
@@ -86,4 +89,4 @@ jobs:
           for ((i=0; i < 3; i++)); do
             create_invalidation "$PATHS" && break || echo "Invalidation failed, retrying in 5 seconds..."
             sleep 5
-          done
+          done
diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml
@@ -37,6 +37,6 @@ jobs:
     - name: validate postoverflows against schema
       run: |
         for ITEM in ./postoverflows/*/*/*.json; do echo $ITEM && ~/go/bin/jv crowdsec-yaml-schemas/parser_schema.json $ITEM ; done
-    - name: validate collections against schema
-      run: |
-        for ITEM in ./collections/*/*.json; do echo $ITEM && ~/go/bin/jv crowdsec-yaml-schemas/collection_schema.json $ITEM ; done
+#    - name: validate collections against schema
+#      run: |
+#        for ITEM in ./collections/*/*.json; do echo $ITEM && ~/go/bin/jv crowdsec-yaml-schemas/collection_schema.json $ITEM ; done
diff --git a/.index.json b/.index.json
@@ -2025,7 +2025,7 @@
   },
   "crowdsecurity/base-http-scenarios": {
    "path": "collections/crowdsecurity/base-http-scenarios.yaml",
-   "version": "0.6",
+   "version": "0.7",
    "versions": {
     "0.1": {
      "digest": "7ee043a9d2e063cad751e6ce5d048f02518a76d39ec81aebed3bae736b0ced9e",
@@ -2050,10 +2050,14 @@
     "0.6": {
      "digest": "2d70781df8c630d36e5f4800bde77dd7e130481e9c658aa0b3aae7ae95e15271",
      "deprecated": false
+    },
+    "0.7": {
+     "digest": "539db14da32a19da683fcfd9c0c92263be5b463e037a3ce35851039c8b512f08",
+     "deprecated": false
     }
    },
    "long_description": "Kipjb250YWlucyBubyBwYXJzZXIsIG1lYW50IHRvIGJlIGVtYmVkZGVkKioKCkEgY29sbGVjdGlvbiBvZiBkZWZlbnNpdmUgKGltcGxlbWVudGF0aW9uIGluZGVwZW5kZW50KSBzY2VuYXJpb3MgZm9yIGh0dHAgc2VydmljZXMgOgogLSBhZ2dyZXNzaXZlIGNyYXdsIGRldGVjdGlvbgogLSBzY2FubmluZy9wcm9iaW5nIGRldGVjdGlvbgogLSBiYWQgdXNlci1hZ2VudCBkZXRlY3Rpb24KIC0gcGF0aCB0cmF2ZXJzYWwgZGV0ZWN0aW9uCiAtIHNlbnNpdGl2ZSBkYXRhIGFjY2VzcyBhdHRlbXB0cyBkZXRlY3Rpb24KIC0gU1FMIGluamVjdGlvbiBkZXRlY3Rpb24KCjp3YXJuaW5nOiBUaGlzIGNvbGxlY3Rpb24gaXMgX25vdF8gYSBXQUYgYW5kIHRoaXMgc2NlbmFyaW8gZG9lcyBfbm90XyBhaW1zIGF0IHJlcGxhY2luZyBhIFdBRi4KCgoK",
-   "content": "cGFyc2VyczoKICAtIGNyb3dkc2VjdXJpdHkvaHR0cC1sb2dzCnNjZW5hcmlvczoKICAtIGNyb3dkc2VjdXJpdHkvaHR0cC1jcmF3bC1ub25fc3RhdGljcwogIC0gY3Jvd2RzZWN1cml0eS9odHRwLXByb2JpbmcKICAtIGNyb3dkc2VjdXJpdHkvaHR0cC1iYWQtdXNlci1hZ2VudAogIC0gY3Jvd2RzZWN1cml0eS9odHRwLXBhdGgtdHJhdmVyc2FsLXByb2JpbmcKICAtIGNyb3dkc2VjdXJpdHkvaHR0cC1zZW5zaXRpdmUtZmlsZXMKICAtIGNyb3dkc2VjdXJpdHkvaHR0cC1zcWxpLXByb2JpbmcKICAtIGNyb3dkc2VjdXJpdHkvaHR0cC14c3MtcHJvYmluZwogIC0gY3Jvd2RzZWN1cml0eS9odHRwLWJhY2tkb29ycy1hdHRlbXB0cwogIC0gbHRzaWNoL2h0dHAtdzAwdHcwMHQKICAtIGNyb3dkc2VjdXJpdHkvaHR0cC1nZW5lcmljLWJmCiAgLSBjcm93ZHNlY3VyaXR5L2h0dHAtb3Blbi1wcm94eQpjb2xsZWN0aW9uczoKICAtIGNyb3dkc2VjdXJpdHkvaHR0cC1jdmUKCmRlc2NyaXB0aW9uOiAiaHR0cCBjb21tb24gOiBzY2FubmVycyBkZXRlY3Rpb24iCmF1dGhvcjogY3Jvd2RzZWN1cml0eQp0YWdzOgogIC0gbGludXgKICAtIGh0dHAKICAtIGNyYXdsCiAgLSBzY2FuCgo=",
+   "content": "cGFyc2VyczoKICAtIGNyb3dkc2VjdXJpdHkvaHR0cC1sb2dzCnNjZW5hcmlvczoKICAtIGNyb3dkc2VjdXJpdHkvaHR0cC1jcmF3bC1ub25fc3RhdGljcwogIC0gY3Jvd2RzZWN1cml0eS9odHRwLXByb2JpbmcKICAtIGNyb3dkc2VjdXJpdHkvaHR0cC1iYWQtdXNlci1hZ2VudAogIC0gY3Jvd2RzZWN1cml0eS9odHRwLXBhdGgtdHJhdmVyc2FsLXByb2JpbmcKICAtIGNyb3dkc2VjdXJpdHkvaHR0cC1zZW5zaXRpdmUtZmlsZXMKICAtIGNyb3dkc2VjdXJpdHkvaHR0cC1zcWxpLXByb2JpbmcKICAtIGNyb3dkc2VjdXJpdHkvaHR0cC14c3MtcHJvYmluZwogIC0gY3Jvd2RzZWN1cml0eS9odHRwLWJhY2tkb29ycy1hdHRlbXB0cwogIC0gbHRzaWNoL2h0dHAtdzAwdHcwMHQKICAtIGNyb3dkc2VjdXJpdHkvaHR0cC1nZW5lcmljLWJmCiAgLSBjcm93ZHNlY3VyaXR5L2h0dHAtb3Blbi1wcm94eQpjb2xsZWN0aW9uczoKICAtIGNyb3dkc2VjdXJpdHkvaHR0cC1jdmUKY29udGV4dHM6CiAgLSBjcm93ZHNlY3VyaXR5L2h0dHBfYmFzZQpkZXNjcmlwdGlvbjogImh0dHAgY29tbW9uIDogc2Nhbm5lcnMgZGV0ZWN0aW9uIgphdXRob3I6IGNyb3dkc2VjdXJpdHkKdGFnczoKICAtIGxpbnV4CiAgLSBodHRwCiAgLSBjcmF3bAogIC0gc2NhbgoK",
    "description": "http common : scanners detection",
    "author": "crowdsecurity",
    "labels": null,
@@ -2073,6 +2077,9 @@
     "crowdsecurity/http-generic-bf",
     "crowdsecurity/http-open-proxy"
    ],
+   "contexts": [
+    "crowdsecurity/http_base"
+   ],
    "collections": [
     "crowdsecurity/http-cve"
    ]
@@ -3137,7 +3144,7 @@
   },
   "crowdsecurity/sshd": {
    "path": "collections/crowdsecurity/sshd.yaml",
-   "version": "0.2",
+   "version": "0.3",
    "versions": {
     "0.1": {
      "digest": "21159aeb87529efcf1a5033f720413d5321a6451bab679a999f7f01a7aa972b3",
@@ -3146,10 +3153,14 @@
     "0.2": {
      "digest": "72f6329808fafbb42da52cc6476a6e794d0a1ae5b3847e0060cf23593dd40352",
      "deprecated": false
+    },
+    "0.3": {
+     "digest": "31d549124634df1d13e67f0903b10c1816690589f4d6add6fec0ed74d30499bb",
+     "deprecated": false
     }
    },
    "long_description": "IyMgU1NIRCBjb2xsZWN0aW9uCgpBIGNvbGxlY3Rpb24gdG8gZGVmZW5kIHNzaGQgYWdhaW5zdCBjb21tb24gYXR0YWNrcyA6CiAtIHNzaCBwYXJzZXIKIC0gc3NoIGJydXRlZm9yY2UgJiBlbnVtZXJhdGlvbiBkZXRlY3Rpb24KIC0gc3NoICdzbG93JyBicnV0ZWZvcmNlICYgZW51bWVyYXRpb24gZGV0ZWN0aW9uCgojIyBBY3F1aXNpdGlvbiB0ZW1wbGF0ZQoKRXhhbXBsZSBhY3F1aXNpdGlvbiBmb3IgdGhpcyBjb2xsZWN0aW9uIDoKCmBgYHlhbWwKZmlsZW5hbWVzOgogIC0gL3Zhci9sb2cvYXV0aC5sb2cKbGFiZWxzOgogIHR5cGU6IHN5c2xvZwpgYGAKCgpub3RlcyA6CiAtICBJZiB5b3UgYXJlIHVzaW5nIGBzeXNsb2dgLCBzZXQgdHlwZSB0byBgc3lzbG9nYCBpbnN0ZWFkCiAtICBEZXBlbmRpbmcgb24geW91ciBkaXN0cmlidXRpb24vT1MsIHBhdGhzIHRvIGxvZyBmaWxlcyBtaWdodCBjaGFuZ2UKIC0gIE9ubHkgcmVsZXZhbnQgaWYgeW91IGFyZSBtYW51YWxseSBpbnN0YWxsaW5nIGNvbGxlY3Rpb24KCg==",
-   "content": "cGFyc2VyczoKICAtIGNyb3dkc2VjdXJpdHkvc3NoZC1sb2dzCnNjZW5hcmlvczoKICAtIGNyb3dkc2VjdXJpdHkvc3NoLWJmCiAgLSBjcm93ZHNlY3VyaXR5L3NzaC1zbG93LWJmCmRlc2NyaXB0aW9uOiAic3NoZCBzdXBwb3J0IDogcGFyc2VyIGFuZCBicnV0ZS1mb3JjZSBkZXRlY3Rpb24iCmF1dGhvcjogY3Jvd2RzZWN1cml0eQp0YWdzOgogIC0gbGludXgKICAtIHNzaAogIC0gYnJ1dGVmb3JjZQoK",
+   "content": "cGFyc2VyczoKICAtIGNyb3dkc2VjdXJpdHkvc3NoZC1sb2dzCnNjZW5hcmlvczoKICAtIGNyb3dkc2VjdXJpdHkvc3NoLWJmCiAgLSBjcm93ZHNlY3VyaXR5L3NzaC1zbG93LWJmCmRlc2NyaXB0aW9uOiAic3NoZCBzdXBwb3J0IDogcGFyc2VyIGFuZCBicnV0ZS1mb3JjZSBkZXRlY3Rpb24iCmNvbnRleHRzOgogIC0gY3Jvd2RzZWN1cml0eS9iZl9iYXNlCmF1dGhvcjogY3Jvd2RzZWN1cml0eQp0YWdzOgogIC0gbGludXgKICAtIHNzaAogIC0gYnJ1dGVmb3JjZQoK",
    "description": "sshd support : parser and brute-force detection",
    "author": "crowdsecurity",
    "labels": null,
@@ -3159,6 +3170,9 @@
    "scenarios": [
     "crowdsecurity/ssh-bf",
     "crowdsecurity/ssh-slow-bf"
+   ],
+   "contexts": [
+    "crowdsecurity/bf_base"
    ]
   },
   "crowdsecurity/sshd-impossible-travel": {
@@ -4082,6 +4096,51 @@
    ]
   }
  },
+ "contexts": {
+  "crowdsecurity/appsec_base": {
+   "path": "contexts/crowdsecurity/appsec_base.yaml",
+   "version": "0.1",
+   "versions": {
+    "0.1": {
+     "digest": "df177378b9b01c6c8b67ff5085eda9325c67b337e31d60c4ea95f743783a5e24",
+     "deprecated": false
+    }
+   },
+   "content": "Y29udGV4dDoKICBydWxlczoKICAtIGV2dC5NZXRhLnJ1bGVfbmFtZQo=",
+   "author": "crowdsecurity",
+   "labels": null
+  },
+  "crowdsecurity/bf_base": {
+   "path": "contexts/crowdsecurity/bf_base.yaml",
+   "version": "0.1",
+   "versions": {
+    "0.1": {
+     "digest": "5b5d0f412ea7da0712fd8e298e9a03642051591adee3817ae529fafa6b66995c",
+     "deprecated": false
+    }
+   },
+   "content": "I2EgZ2VuZXJpYyBjb250ZXh0IGZvciBicnV0ZWZvcmNlIGJhc2VkIHNjZW5hcmlvcwpjb250ZXh0OgogIHRhcmdldF91c2VyOgogICAgLSBldnQuTWV0YS50YXJnZXRfdXNlcgo=",
+   "author": "crowdsecurity",
+   "labels": null
+  },
+  "crowdsecurity/http_base": {
+   "path": "contexts/crowdsecurity/http_base.yaml",
+   "version": "0.2",
+   "versions": {
+    "0.1": {
+     "digest": "a8f832e367aa06576e6c552e839b5e61bedfcb8098bd4049c6a0dff06ecab810",
+     "deprecated": false
+    },
+    "0.2": {
+     "digest": "d0f465d5ff866a91637cd59bc9a18f881bbebf03f8360be9df8182035c927909",
+     "deprecated": false
+    }
+   },
+   "content": "I3RoaXMgY29udGV4dCBmaWxlIGlzIGludGVuZGVkIHRvIHByb3ZpZGUgbWluaW1hbCBhbmQgdXNlZnVsIGluZm9ybWF0aW9uIGFib3V0IEhUVFAgc2NlbmFyaW9zLgpjb250ZXh0OgogIHRhcmdldF91cmk6CiAgLSBldnQuTWV0YS5odHRwX3BhdGgKICB1c2VyX2FnZW50OgogIC0gZXZ0Lk1ldGEuaHR0cF91c2VyX2FnZW50CiAgbWV0aG9kOgogIC0gZXZ0Lk1ldGEuaHR0cF92ZXJiCiAgc3RhdHVzOgogICAgLSBldnQuTWV0YS5odHRwX3N0YXR1cwo=",
+   "author": "crowdsecurity",
+   "labels": null
+  }
+ },
  "parsers": {
   "Dominic-Wagner/vaultwarden-logs": {
    "path": "parsers/s01-parse/Dominic-Wagner/vaultwarden-logs.yaml",

diff --git a/ci.go b/ci.go
@@ -28,6 +28,7 @@ type typeInfo struct {
 	Scenarios       []string               `json:"scenarios,omitempty"`
 	AppsecRules     []string               `json:"appsec-rules,omitempty"`
 	AppsecConfigs   []string               `json:"appsec-configs,omitempty"`
+	Contexts        []string               `json:"contexts,omitempty"`
 	Collections     []string               `json:"collections,omitempty"`
 }
 
@@ -41,6 +42,7 @@ type fileInfo struct {
 	Scenarios     []string               `yaml:"scenarios,omitempty"`
 	AppsecRules   []string               `yaml:"appsec-rules,omitempty"`
 	AppsecConfigs []string               `yaml:"appsec-configs,omitempty"`
+	Contexts      []string               `yaml:"contexts,omitempty"`
 	Collections   []string               `yaml:"collections,omitempty"`
 }
 
@@ -55,6 +57,7 @@ var types = []string{
 	"postoverflows",
 	"appsec-rules",
 	"appsec-configs",
+	"contexts",
 	"collections",
 }
 

diff --git a/collections/crowdsecurity/base-http-scenarios.yaml b/collections/crowdsecurity/base-http-scenarios.yaml
@@ -14,7 +14,8 @@ scenarios:
   - crowdsecurity/http-open-proxy
 collections:
   - crowdsecurity/http-cve
-
+contexts:
+  - crowdsecurity/http_base
 description: "http common : scanners detection"
 author: crowdsecurity
 tags:

diff --git a/collections/crowdsecurity/sshd.yaml b/collections/crowdsecurity/sshd.yaml
@@ -4,6 +4,8 @@ scenarios:
   - crowdsecurity/ssh-bf
   - crowdsecurity/ssh-slow-bf
 description: "sshd support : parser and brute-force detection"
+contexts:
+  - crowdsecurity/bf_base
 author: crowdsecurity
 tags:
   - linux

diff --git a/contexts/crowdsecurity/appsec_base.yaml b/contexts/crowdsecurity/appsec_base.yaml
@@ -0,0 +1,3 @@
+context:
+  rules:
+  - evt.Meta.rule_name
diff --git a/contexts/crowdsecurity/bf_base.yaml b/contexts/crowdsecurity/bf_base.yaml
@@ -0,0 +1,4 @@
+#a generic context for bruteforce based scenarios
+context:
+  target_user:
+    - evt.Meta.target_user
diff --git a/contexts/crowdsecurity/http_base.yaml b/contexts/crowdsecurity/http_base.yaml
@@ -0,0 +1,10 @@
+#this context file is intended to provide minimal and useful information about HTTP scenarios.
+context:
+  target_uri:
+  - evt.Meta.http_path
+  user_agent:
+  - evt.Meta.http_user_agent
+  method:
+  - evt.Meta.http_verb
+  status:
+    - evt.Meta.http_status
diff --git a/generate.go b/generate.go
@@ -17,6 +17,7 @@ const (
 	PARSER_TYPE         = "parsers"
 	SCENARIO_TYPE       = "scenarios"
 	POSTOVERFLOW_TYPE   = "postoverflows"
+	CONTEXT_TYPE        = "contexts"
 	APPSEC_RULES_TYPE   = "appsec-rules"
 	APPSEC_CONFIGS_TYPE = "appsec-configs"
 	COLLECTIONS_TYPE    = "collections"
@@ -48,7 +49,7 @@ func (ti *typeInfo) generate(filepath string, configType string) (string, error)
 		user = pathSplit[1]
 		configName = pathSplit[2]
 		configName = strings.Split(configName, ".")[0]
-	case SCENARIO_TYPE, APPSEC_RULES_TYPE, APPSEC_CONFIGS_TYPE, COLLECTIONS_TYPE:
+	case SCENARIO_TYPE, APPSEC_RULES_TYPE, APPSEC_CONFIGS_TYPE, COLLECTIONS_TYPE, CONTEXT_TYPE:
 		if len(pathSplit) != 2 {
 			return "", fmt.Errorf("invalid filepath '%s', should be : './%s/<user>/<scenario.yaml>'", configType, filepath)
 		}
@@ -129,6 +130,11 @@ func (ti *typeInfo) generate(filepath string, configType string) (string, error)
 		} else {
 			ti.AppsecConfigs = nil
 		}
+		if len(fInfo.Contexts) > 0 {
+			ti.Contexts = fInfo.Contexts
+		} else {
+			ti.Contexts = nil
+		}
 	}
 
 	// versions informations (digest and deprecated for each version)