Skip to content

Commit

Permalink
Update elastic/logs integrations to 8.13.3 (elastic#613)
Browse files Browse the repository at this point in the history 
Update integrations for elastic/logs to 8.13.3
  • Loading branch information
@ebadyano
ebadyano committed Jul 15, 2024
1 parent b580277 commit f018785
Show file tree
Hide file tree
Showing 65 changed files with 16,563 additions and 13,042 deletions.
34 changes: 20 additions & 14 deletions elastic/logs/pipelines/.fleet_final_pipeline-1.json
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,19 @@
{
"version": 1,
"version": 4,
"_meta": {
"managed_by": "fleet",
"managed": true
},
"description": "Final pipeline for processing all incoming Fleet Agent documents.",
"processors": [
{
"set": {
"description": "Add time when event was ingested.",
"field": "event.ingested",
"copy_from": "_ingest.timestamp"
}
},
{
"script": {
"description": "Remove sub-seconds from event.ingested to improve storage efficiency.",
"description": "Add time when event was ingested (and remove sub-seconds to improve storage efficiency)",
"tag": "truncate-subseconds-event-ingested",
"source": "ctx.event.ingested = ctx.event.ingested.withNano(0).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);",
"ignore_failure": true
}
},
"ignore_failure": true,
"source": "if (ctx?.event == null) { ctx.event = [:]; } ctx.event.ingested = metadata().now.withNano(0).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);"
}
},
{
"remove": {
"description": "Remove any pre-existing untrusted values.",
Expand All @@ -27,6 +24,15 @@
"ignore_missing": true
}
},
{
"remove": {
"description": "Remove event.original unless the preserve_original_event tag is set",
"field": "event.original",
"if": "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))",
"ignore_failure": true,
"ignore_missing": true
}
},
{
"set_security_user": {
"field": "_security",
Expand Down Expand Up @@ -85,4 +91,4 @@
}
}
]
}
}
289 changes: 289 additions & 0 deletions elastic/logs/pipelines/logs-apache.access-1.18.0.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,289 @@
{
"description": "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins.",
"processors": [
{
"pipeline": {
"if": "ctx.message.startsWith('{')",
"name": "logs-apache.access-1.18.0-third-party"
}
},
{
"set": {
"field": "event.ingested",
"value": "{{_ingest.timestamp}}"
}
},
{
"set": {
"field": "ecs.version",
"value": "8.5.1"
}
},
{
"rename": {
"field": "message",
"target_field": "event.original",
"ignore_missing": true,
"if": "ctx.event?.original == null"
}
},
{
"remove": {
"field": "message",
"ignore_missing": true,
"if": "ctx.event?.original != null",
"description": "The `message` field is no longer required if the document has an `event.original` field."
}
},
{
"grok": {
"field": "event.original",
"patterns": [
"(%{IPORHOST:destination.domain} )?%{IPORHOST:source.address} %{DATA:apache.access.identity} %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?( X-Forwarded-For=\"%{ADDRESS_LIST:apache.access.remote_addresses}\")?",
"%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"-\" %{NUMBER:http.response.status_code:long} -",
"\\[%{HTTPDATE:apache.access.time}\\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} \"%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}\" (-|%{NUMBER:http.response.body.bytes:long})"
],
"ignore_missing": true,
"pattern_definitions": {
"ADDRESS_LIST": "(%{IP})(\"?,?\\s*(%{IP}))*"
}
}
},
{
"split": {
"field": "apache.access.remote_addresses",
"separator": "\"?,\\s*",
"ignore_missing": true
}
},
{
"set": {
"field": "network.forwarded_ip",
"value": "{{{apache.access.remote_addresses.0}}}",
"if": "ctx.apache?.access?.remote_addresses != null && ctx.apache.access.remote_addresses.length > 0"
}
},
{
"script": {
"if": "ctx.apache?.access?.remote_addresses != null && ctx.apache.access.remote_addresses.length > 0",
"lang": "painless",
"tag": "Get source address",
"description": "Extract from remote_addresses, the first non-private IP to ctx.client.ip",
"source": "boolean isPrivateCIDR(def ip) {\n CIDR class_a_network = new CIDR('10.0.0.0/8');\n CIDR class_b_network = new CIDR('172.16.0.0/12');\n CIDR class_c_network = new CIDR('192.168.0.0/16');\n\n try {\n return class_a_network.contains(ip) || class_b_network.contains(ip) || class_c_network.contains(ip);\n } catch (IllegalArgumentException e) {\n return false;\n }\n}\ntry {\n if (ctx.client == null) {\n Map map = new HashMap();\n ctx.put(\"client\", map);\n }\n\n def found = false;\n for (def item : ctx.apache.access.remote_addresses) {\n if (!isPrivateCIDR(item)) {\n ctx.client.ip = item;\n found = true;\n break;\n }\n }\n if (!found) {\n ctx.client.ip = ctx.apache.access.remote_addresses[0];\n }\n} catch (Exception e) {\n ctx.client.ip = null;\n}"
}
},
{
"append": {
"field": "apache.access.remote_addresses",
"value": [
"{{source.address}}"
],
"if": "ctx.source?.address != null"
}
},
{
"uri_parts": {
"field": "_tmp.url_orig",
"ignore_failure": true
}
},
{
"remove": {
"field": [
"_tmp"
],
"ignore_missing": true
}
},
{
"set": {
"field": "url.domain",
"value": "{{destination.domain}}",
"if": "ctx.url?.domain == null && ctx.destination?.domain != null"
}
},
{
"set": {
"field": "event.kind",
"value": "event"
}
},
{
"append": {
"field": "event.category",
"value": "web"
}
},
{
"set": {
"field": "event.outcome",
"value": "success",
"if": "ctx.http?.response?.status_code != null && ctx.http.response.status_code < 400"
}
},
{
"set": {
"field": "event.outcome",
"value": "failure",
"if": "ctx.http?.response?.status_code != null && ctx.http.response.status_code > 399"
}
},
{
"grok": {
"field": "source.address",
"ignore_missing": true,
"patterns": [
"^(%{IP:source.ip}|%{HOSTNAME:source.domain})$"
]
}
},
{
"remove": {
"field": "event.created",
"ignore_missing": true,
"ignore_failure": true
}
},
{
"rename": {
"field": "@timestamp",
"target_field": "event.created"
}
},
{
"date": {
"field": "apache.access.time",
"target_field": "@timestamp",
"formats": [
"dd/MMM/yyyy:H:m:s Z"
],
"ignore_failure": true
}
},
{
"remove": {
"field": "apache.access.time",
"ignore_failure": true
}
},
{
"user_agent": {
"field": "user_agent.original",
"ignore_failure": true
}
},
{
"geoip": {
"field": "source.ip",
"target_field": "source.geo",
"ignore_missing": true
}
},
{
"geoip": {
"database_file": "GeoLite2-ASN.mmdb",
"field": "source.ip",
"target_field": "source.as",
"properties": [
"asn",
"organization_name"
],
"ignore_missing": true
}
},
{
"rename": {
"field": "source.as.asn",
"target_field": "source.as.number",
"ignore_missing": true
}
},
{
"rename": {
"field": "source.as.organization_name",
"target_field": "source.as.organization.name",
"ignore_missing": true
}
},
{
"set": {
"field": "tls.cipher",
"value": "{{apache.access.ssl.cipher}}",
"if": "ctx.apache?.access?.ssl?.cipher != null"
}
},
{
"script": {
"lang": "painless",
"if": "ctx.apache?.access?.ssl?.protocol != null",
"source": "def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken(\"v\"); if (parts.length != 2) {\n return;\n} if (parts[1].contains(\".\")) {\n ctx.tls.version = parts[1];\n} else {\n ctx.tls.version = parts[1] + \".0\";\n} ctx.tls.version_protocol = parts[0];"
}
},
{
"script": {
"lang": "painless",
"description": "This script processor iterates over the whole document to remove fields with null values.",
"source": "void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null);\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n"
}
},
{
"remove": {
"field": "event.original",
"if": "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))",
"ignore_failure": true,
"ignore_missing": true
}
},
{
"remove": {
"field": "apache.access.identity",
"if": "ctx.apache?.access?.identity == \"-\"",
"ignore_failure": true,
"ignore_missing": true
}
},
{
"pipeline": {
"name": "global@custom",
"ignore_missing_pipeline": true,
"description": "[Fleet] Global pipeline for all data streams"
}
},
{
"pipeline": {
"name": "logs@custom",
"ignore_missing_pipeline": true,
"description": "[Fleet] Pipeline for all data streams of type `logs`"
}
},
{
"pipeline": {
"name": "logs-apache.integration@custom",
"ignore_missing_pipeline": true,
"description": "[Fleet] Pipeline for all data streams of type `logs` defined by the `apache` integration"
}
},
{
"pipeline": {
"name": "logs-apache.access@custom",
"ignore_missing_pipeline": true,
"description": "[Fleet] Pipeline for the `apache.access` dataset"
}
}
],
"on_failure": [
{
"set": {
"field": "error.message",
"value": "{{ _ingest.on_failure_message }}"
}
}
],
"_meta": {
"managed_by": "fleet",
"managed": true,
"package": {
"name": "apache"
}
}
}
Loading

0 comments on commit f018785

Please sign in to comment.